Run tests in parallel (#1657)

kuzudb · Jun 12, 2023 · 0666fd4 · 0666fd4
1 parent cd3f679
commit 0666fd4
Show file tree

Hide file tree

Showing 20 changed files with 136 additions and 84 deletions.
diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml
@@ -64,7 +64,7 @@ jobs:
         run: CC=gcc CXX=g++ make alldebug NUM_THREADS=32
 
       - name: Run test with ASan
-        run: ctest
+        run: ctest --output-on-failure -j 10
         env:
           LD_PRELOAD: "/usr/lib/x86_64-linux-gnu/libasan.so.6"
           ASAN_OPTIONS: "detect_leaks=1:log_path=/tmp/asan.log"

diff --git a/.gitignore b/.gitignore
@@ -16,7 +16,8 @@ __pycache__/
 *.py[cod]
 *$py.class
 cmake-build-debug/
-test/unittest_temp/
+test/unittest_temp_*
+dataset/parquet_temp_*
 tools/python_api/test/test_PYTHON_CSV.csv
 
 # antlr4 jar

diff --git a/Makefile b/Makefile
@@ -3,13 +3,18 @@
 GENERATOR=
 FORCE_COLOR=
 NUM_THREADS=
+TEST_JOBS=
 SANITIZER_FLAG=
 ROOT_DIR=$(CURDIR)
 
 ifndef $(NUM_THREADS)
 	NUM_THREADS=1
 endif
 
+ifndef $(TEST_JOBS)
+	TEST_JOBS=10
+endif
+
 ifeq ($(OS),Windows_NT)
 	ifndef $(GEN)
 		GEN=ninja
@@ -83,14 +88,14 @@ test: arrow
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE ../.. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 	cd $(ROOT_DIR)/build/release/test && \
-	ctest --output-on-failure
+	ctest --output-on-failure -j ${TEST_JOBS}
 
 lcov: arrow
 	$(call mkdirp,build/release) && cd build/release && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE -DBUILD_NODEJS=TRUE -DBUILD_LCOV=TRUE ../.. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 	cd $(ROOT_DIR)/build/release/test && \
-	ctest --output-on-failure
+	ctest --output-on-failure -j ${TEST_JOBS}
 
 pytest: arrow
 	$(MAKE) release

diff --git a/src/storage/store/nodes_statistics_and_deleted_ids.cpp b/src/storage/store/nodes_statistics_and_deleted_ids.cpp
@@ -130,17 +130,16 @@ void NodeStatisticsAndDeletedIDs::errorIfNodeHasEdges(offset_t nodeOffset) {
         if (numElementsInList != 0) {
             throw RuntimeException(StringUtils::string_format(
                 "Currently deleting a node with edges is not supported. node table {} nodeOffset "
-                "{} has {} (one-to-many or many-to-many) edges for edge file: {}.",
-                tableID, nodeOffset, numElementsInList,
-                adjList->getFileHandle()->getFileInfo()->path.c_str()));
+                "{} has {} (one-to-many or many-to-many) edges.",
+                tableID, nodeOffset, numElementsInList));
         }
     }
     for (Column* adjColumn : adjListsAndColumns.second) {
         if (!adjColumn->isNull(nodeOffset, transaction::Transaction::getDummyWriteTrx().get())) {
             throw RuntimeException(StringUtils::string_format(
                 "Currently deleting a node with edges is not supported. node table {} nodeOffset "
-                "{}  has a 1-1 edge for edge file: {}.",
-                tableID, nodeOffset, adjColumn->getFileHandle()->getFileInfo()->path.c_str()));
+                "{}  has a 1-1 edge.",
+                tableID, nodeOffset));
         }
     }
 }

diff --git a/test/c_api/database_test.cpp b/test/c_api/database_test.cpp
@@ -2,11 +2,10 @@
 
 class CApiDatabaseTest : public EmptyDBTest {
 public:
-    void SetUp() override {}
+    void SetUp() override { EmptyDBTest::SetUp(); }
 };
 
 TEST_F(CApiDatabaseTest, CreationAndDestroy) {
-    auto databasePath = TestHelper::getTmpTestDir();
     auto databasePathCStr = databasePath.c_str();
     auto database = kuzu_database_init(databasePathCStr, 0);
     ASSERT_NE(database, nullptr);

diff --git a/test/c_api/query_result_test.cpp b/test/c_api/query_result_test.cpp
@@ -130,7 +130,7 @@ TEST_F(CApiQueryResultTest, WriteToCSV) {
     auto connection = getConnection();
     auto result = kuzu_connection_query(connection, query);
     ASSERT_TRUE(kuzu_query_result_is_success(result));
-    auto outputPath = TestHelper::getTmpTestDir() + "/output_CSV_CAPI.csv";
+    auto outputPath = databasePath + "/output_CSV_CAPI.csv";
     kuzu_query_result_write_to_csv(result, outputPath.c_str(), ',', '"', '\n');
     std::ifstream f(outputPath);
     std::ostringstream ss;

diff --git a/test/include/graph_test/graph_test.h b/test/include/graph_test/graph_test.h
@@ -26,15 +26,15 @@ class BaseGraphTest : public Test {
     void SetUp() override {
         systemConfig = std::make_unique<main::SystemConfig>(
             common::BufferPoolConstants::DEFAULT_BUFFER_POOL_SIZE_FOR_TESTING);
-        if (common::FileUtils::fileOrPathExists(TestHelper::getTmpTestDir())) {
-            common::FileUtils::removeDir(TestHelper::getTmpTestDir());
+        setDatabasePath();
+        if (common::FileUtils::fileOrPathExists(databasePath)) {
+            common::FileUtils::removeDir(databasePath);
         }
-        databasePath = TestHelper::getTmpTestDir();
     }
 
     virtual std::string getInputDir() = 0;
 
-    void TearDown() override { common::FileUtils::removeDir(TestHelper::getTmpTestDir()); }
+    void TearDown() override { common::FileUtils::removeDir(databasePath); }
 
     void createDBAndConn();
 
@@ -133,7 +133,20 @@ class BaseGraphTest : public Test {
     void commitOrRollbackConnectionAndInitDBIfNecessary(
         bool isCommit, TransactionTestType transactionTestType);
 
+    inline std::string getTestGroupAndName() {
+        const ::testing::TestInfo* const testInfo =
+            ::testing::UnitTest::GetInstance()->current_test_info();
+        return std::string(testInfo->test_case_name()) + "." + std::string(testInfo->name());
+    }
+
 private:
+    void setDatabasePath() {
+        const ::testing::TestInfo* const testInfo =
+            ::testing::UnitTest::GetInstance()->current_test_info();
+        databasePath = TestHelper::appendKuzuRootPath(
+            TestHelper::TMP_TEST_DIR + getTestGroupAndName() + TestHelper::getMillisecondsSuffix());
+    }
+
     void validateRelPropertyFiles(catalog::RelTableSchema* relTableSchema,
         common::RelDataDirection relDirection, bool isColumnProperty, common::DBFileType dbFileType,
         bool existence);

diff --git a/test/include/test_helper/test_helper.h b/test/include/test_helper/test_helper.h
@@ -40,24 +40,22 @@ class TestHelper {
     static constexpr char E2E_TEST_FILES_DIRECTORY[] = "test/test_files";
     static constexpr char SCHEMA_FILE_NAME[] = "schema.cypher";
     static constexpr char COPY_FILE_NAME[] = "copy.cypher";
-    static constexpr char PARQUET_TEMP_DATASET_PATH[] = "dataset/parquet_temp/";
+    static constexpr char PARQUET_TEMP_DATASET_PATH[] = "dataset/parquet_temp_";
+    static constexpr char TMP_TEST_DIR[] = "test/unittest_temp_";
 
-    static std::string getTmpTestDir() { return appendKuzuRootPath("test/unittest_temp/"); }
     static std::string getTestListFile() {
         return appendKuzuRootPath(std::string(E2E_TEST_FILES_DIRECTORY) + "/test_list");
     }
 
-    static std::string appendParquetDatasetTempDir(const std::string& dataset) {
-        return TestHelper::appendKuzuRootPath(TestHelper::PARQUET_TEMP_DATASET_PATH + dataset);
-    }
-
     static std::string appendKuzuRootPath(const std::string& path) {
         return KUZU_ROOT_DIRECTORY + std::string("/") + path;
     }
 
     static std::unique_ptr<planner::LogicalPlan> getLogicalPlan(
         const std::string& query, Connection& conn);
 
+    static std::string getMillisecondsSuffix();
+
 private:
     static void initializeConnection(TestQueryConfig* config, Connection& conn);
     static bool testQuery(TestQueryConfig* config, Connection& conn);

diff --git a/test/include/test_runner/csv_to_parquet_converter.h b/test/include/test_runner/csv_to_parquet_converter.h
@@ -8,7 +8,13 @@ namespace testing {
 // The dataset directory must contain schema and copy files.
 class CSVToParquetConverter {
 public:
-    static void convertCSVDatasetToParquet(std::string& dataset);
+    static std::string convertCSVDatasetToParquet(
+        const std::string& csvDatasetPath, const std::string& parquetDatasetPath);
+
+    inline static std::string replaceSlashesWithUnderscores(std::string dataset) {
+        std::replace(dataset.begin(), dataset.end(), '/', '_');
+        return dataset;
+    }
 
 private:
     struct CopyCommandInfo {
@@ -20,28 +26,23 @@ class CSVToParquetConverter {
     };
 
     static std::vector<CopyCommandInfo> readCopyCommandsFromCopyCypherFile(
-        const std::string& dataset);
+        const std::string& csvDatasetPath, const std::string& parquetDatasetPath);
 
     static void convertCSVFilesToParquet(
         const std::vector<CSVToParquetConverter::CopyCommandInfo>& copyCommands);
 
     static CopyCommandInfo createCopyCommandInfo(
-        const std::string& dataset, std::string copyStatement);
+        const std::string& parquetDatasetPath, std::string copyStatement);
 
     static arrow::Status runCSVToParquetConversion(const std::string& inputFile,
         const std::string& outputFile, char delimiter, bool hasHeader);
 
     static void copySchema(
         const std::string& csvDatasetPath, const std::string& parquetDatasetPath);
 
-    static void createCopyFile(const std::string& dataset,
+    static void createCopyFile(const std::string& parquetDatasetPath,
         const std::vector<CSVToParquetConverter::CopyCommandInfo>& copyCommands);
 
-    inline static std::string replaceSlashesWithUnderscores(std::string dataset) {
-        std::replace(dataset.begin(), dataset.end(), '/', '_');
-        return dataset;
-    }
-
     static std::string extractPath(std::string& str, char delimiter);
 };
 

diff --git a/test/main/csv_output_test.cpp b/test/main/csv_output_test.cpp
@@ -15,8 +15,8 @@ TEST_F(CSVOutputTest, BasicCSVTest) {
     auto query = "MATCH (a:person)-[:workAt]->(o:organisation) RETURN a.fName, a.gender,"
                  "a.eyeSight, a.birthdate, a.registerTime, o.name";
     auto result = conn->query(query);
-    result->writeToCSV(TestHelper::getTmpTestDir() + "/output_CSV_BASIC.csv");
-    std::ifstream f(TestHelper::getTmpTestDir() + "/output_CSV_BASIC.csv");
+    result->writeToCSV(databasePath + "/output_CSV_BASIC.csv");
+    std::ifstream f(databasePath + "/output_CSV_BASIC.csv");
     std::ostringstream ss;
     ss << f.rdbuf();
     std::string fileString = ss.str();
@@ -37,8 +37,8 @@ TEST_F(CSVOutputTest, ListCSVTest) {
         newline;
     auto query = "MATCH (a:person) RETURN a.usedNames, a.workedHours, [a.fName, a.fName]";
     auto result = conn->query(query);
-    result->writeToCSV(TestHelper::getTmpTestDir() + "/output_CSV_LIST.csv");
-    std::ifstream f(TestHelper::getTmpTestDir() + "/output_CSV_LIST.csv");
+    result->writeToCSV(databasePath + "/output_CSV_LIST.csv");
+    std::ifstream f(databasePath + "/output_CSV_LIST.csv");
     std::ostringstream ss;
     ss << f.rdbuf();
     std::string fileString = ss.str();
@@ -49,8 +49,8 @@ TEST_F(CSVOutputTest, AlternateDelimCSVTest) {
     std::string listOutput = R"(ABFsUni	"-2"-CsWork	"-100"-DEsWork	7-)";
     auto query = "MATCH (o:organisation) RETURN o.name, o.score";
     auto result = conn->query(query);
-    result->writeToCSV(TestHelper::getTmpTestDir() + "/output_CSV_LIST.csv", '\t', '"', '-');
-    std::ifstream f(TestHelper::getTmpTestDir() + "/output_CSV_LIST.csv");
+    result->writeToCSV(databasePath + "/output_CSV_LIST.csv", '\t', '"', '-');
+    std::ifstream f(databasePath + "/output_CSV_LIST.csv");
     std::ostringstream ss;
     ss << f.rdbuf();
     std::string fileString = ss.str();
@@ -66,8 +66,8 @@ TEST_F(CSVOutputTest, AlternateEscapeCSVTest) {
         R"(`[10,11,12,3,4,5,6,7]`,Hubert Blaine Wolfeschlegelsteinhausenbergerdorff)" + newline;
     auto query = "MATCH (p:person) RETURN p.workedHours, p.fName";
     auto result = conn->query(query);
-    result->writeToCSV(TestHelper::getTmpTestDir() + "/output_CSV_LIST.csv", ',', '`');
-    std::ifstream f(TestHelper::getTmpTestDir() + "/output_CSV_LIST.csv");
+    result->writeToCSV(databasePath + "/output_CSV_LIST.csv", ',', '`');
+    std::ifstream f(databasePath + "/output_CSV_LIST.csv");
     std::ostringstream ss;
     ss << f.rdbuf();
     std::string fileString = ss.str();

diff --git a/test/runner/CMakeLists.txt b/test/runner/CMakeLists.txt
@@ -6,3 +6,4 @@ add_kuzu_test(e2e_set_transaction_test e2e_set_transaction_test.cpp)
 add_kuzu_test(e2e_update_rel_test e2e_update_rel_test.cpp)
 add_kuzu_test(e2e_delete_rel_test e2e_delete_rel_test.cpp)
 add_kuzu_test(e2e_create_rel_test e2e_create_rel_test.cpp)
+add_kuzu_test(cleanup_test cleanup_test.cpp)
diff --git a/test/runner/cleanup_test.cpp b/test/runner/cleanup_test.cpp
@@ -0,0 +1,29 @@
+#include "graph_test/graph_test.h"
+
+using namespace kuzu::testing;
+using namespace kuzu::common;
+
+void deleteMatchingDir(const std::string& dirPath, const std::string& match) {
+    for (const auto& entry : std::filesystem::directory_iterator(dirPath)) {
+        if (entry.path().filename().string().find(match) != std::string::npos) {
+            FileUtils::removeDir(entry.path().string());
+        }
+    }
+}
+
+int main(int argc, char** argv) {
+    std::vector<std::string> tempDirs = {
+        TestHelper::PARQUET_TEMP_DATASET_PATH, TestHelper::TMP_TEST_DIR};
+    if (argc > 1 && std::string(argv[1]) == "--gtest_list_tests") {
+        for (const auto& tempDir : tempDirs) {
+            // path = test/unittest_temp_
+            std::filesystem::path path = tempDir;
+            // dirToCheck = test
+            std::string dirToCheck = path.parent_path().string();
+            // dirPatternToRemove = unittest_temp_
+            std::string dirPatternToRemove = path.filename().string();
+            deleteMatchingDir(TestHelper::appendKuzuRootPath(dirToCheck), dirPatternToRemove);
+        }
+    }
+    return 0;
+}
diff --git a/test/runner/e2e_delete_create_transaction_test.cpp b/test/runner/e2e_delete_create_transaction_test.cpp
@@ -304,8 +304,7 @@ TEST_F(DeleteNodeWithEdgesErrorTest, DeleteNodeWithEdgesError) {
     auto result = conn->query("match (p:person) delete p");
     ASSERT_EQ(result->getErrorMessage(),
         "Runtime exception: Currently deleting a node with edges is not supported. node table 0 "
-        "nodeOffset 0 has 1 (one-to-many or many-to-many) edges for edge file: " +
-            TestHelper::appendKuzuRootPath("test/unittest_temp/r-1-0.lists."));
+        "nodeOffset 0 has 1 (one-to-many or many-to-many) edges.");
 }
 
 TEST_F(CreateDeleteInt64NodeTrxTest, MixedInsertDeleteCommitNormalExecution) {

diff --git a/test/runner/e2e_test.cpp b/test/runner/e2e_test.cpp
@@ -7,46 +7,51 @@ using ::testing::Test;
 using namespace kuzu::testing;
 using namespace kuzu::common;
 
-class EndToEndEnvironment : public testing::Environment {
-public:
-    virtual void SetUp() {
-        FileUtils::createDirIfNotExists(
-            TestHelper::appendKuzuRootPath(TestHelper::PARQUET_TEMP_DATASET_PATH));
-    }
-    virtual void TearDown() {
-        FileUtils::removeDir(TestHelper::appendKuzuRootPath(TestHelper::PARQUET_TEMP_DATASET_PATH));
-    }
-};
-
 class EndToEndTest : public DBTest {
 public:
     explicit EndToEndTest(TestGroup::DatasetType datasetType, std::string dataset,
         uint64_t bufferPoolSize, std::vector<std::unique_ptr<TestStatement>> testStatements)
         : datasetType{datasetType}, dataset{dataset}, bufferPoolSize{bufferPoolSize},
           testStatements{std::move(testStatements)} {}
+
     void SetUp() override {
         setUpDataset();
         BaseGraphTest::SetUp();
         systemConfig->bufferPoolSize = bufferPoolSize;
         createDBAndConn();
         initGraph();
     }
+
     void setUpDataset() {
+        parquetTempDatasetPath = generateParquetTempDatasetPath();
+        dataset = TestHelper::appendKuzuRootPath("dataset/" + dataset);
         if (datasetType == TestGroup::DatasetType::CSV_TO_PARQUET) {
-            CSVToParquetConverter::convertCSVDatasetToParquet(dataset);
-        } else {
-            dataset = TestHelper::appendKuzuRootPath("dataset/" + dataset);
+            dataset =
+                CSVToParquetConverter::convertCSVDatasetToParquet(dataset, parquetTempDatasetPath);
         }
     }
-    void TearDown() override { FileUtils::removeDir(TestHelper::getTmpTestDir()); }
-    std::string getInputDir() override { return dataset + "/"; }
+
+    void TearDown() override {
+        FileUtils::removeDir(databasePath);
+        FileUtils::removeDir(parquetTempDatasetPath);
+    }
+
     void TestBody() override { runTest(testStatements); }
+    std::string getInputDir() override { return dataset + "/"; }
 
 private:
     TestGroup::DatasetType datasetType;
     std::string dataset;
+    std::string parquetTempDatasetPath;
     uint64_t bufferPoolSize;
     std::vector<std::unique_ptr<TestStatement>> testStatements;
+
+    const std::string generateParquetTempDatasetPath() {
+        return TestHelper::appendKuzuRootPath(
+            TestHelper::PARQUET_TEMP_DATASET_PATH +
+            CSVToParquetConverter::replaceSlashesWithUnderscores(dataset) + getTestGroupAndName() +
+            TestHelper::getMillisecondsSuffix());
+    }
 };
 
 void parseAndRegisterTestGroup(const std::string& path, bool generateTestList = false) {
@@ -102,7 +107,7 @@ std::string findTestFile(std::string testCase) {
     return "";
 }
 
-void checkCtestParams(int argc, char** argv) {
+void checkGtestParams(int argc, char** argv) {
     if (argc > 1) {
         std::string argument = argv[1];
         if (argument == "--gtest_list_tests") {
@@ -120,9 +125,8 @@ void checkCtestParams(int argc, char** argv) {
 }
 
 int main(int argc, char** argv) {
-    checkCtestParams(argc, argv);
+    checkGtestParams(argc, argv);
     testing::InitGoogleTest(&argc, argv);
-    testing::AddGlobalTestEnvironment(new EndToEndEnvironment);
     if (argc > 1) {
         auto path = TestHelper::appendKuzuRootPath(
             FileUtils::joinPath(TestHelper::E2E_TEST_FILES_DIRECTORY, argv[1]));