separate benchmark and test

.github/workflows/ci-workflow.yml

@@ -58,7 +58,7 @@ jobs:
       - run: pip3 install --user -r tools/python_api/requirements_dev.txt
 
       - name: build
-        run: make release NUM_THREADS=30
+        run: make benchmark NUM_THREADS=30
 
       - name: benchmark
         run: python3 benchmark/benchmark_runner.py --dataset ldbc-sf100 --thread 1

CMakeLists.txt

@@ -43,7 +43,8 @@ endif()
 if(${ENABLE_UBSAN})
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-common -fpermissive")
 endif()
-option(BUILD_TESTS "Build C++ and Python tests and benchmark." FALSE)
+option(BUILD_TESTS "Build C++ and Python tests." FALSE)
+option(BUILD_BENCHMARK "Build benchmarks." FALSE)
 
 function(link_threads LIBRARY)
     if (CMAKE_VERSION VERSION_LESS "3.1")
@@ -56,7 +57,7 @@ endfunction()
 function(add_kuzu_test TEST_NAME)
     set(SRCS ${ARGN})
     add_executable(${TEST_NAME} ${SRCS})
-    target_link_libraries(${TEST_NAME} PRIVATE test_helper)
+    target_link_libraries(${TEST_NAME} PRIVATE test_helper graph_test)
     target_include_directories(${TEST_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/test/include)
     include(GoogleTest)
     gtest_discover_tests(${TEST_NAME})
@@ -85,5 +86,7 @@ add_subdirectory(third_party)
 add_subdirectory(src)
 if (${BUILD_TESTS})
     add_subdirectory(test)
+elseif (${BUILD_BENCHMARK})
+    add_subdirectory(test/test_helper)
 endif()
 add_subdirectory(tools)

Makefile

@@ -47,16 +47,26 @@ debug: arrow
 all: arrow
 	mkdir -p build/release && \
 	cd build/release && \
-	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE ../.. && \
+	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE -DBUILD_BENCHMARK=TRUE ../.. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 
 alldebug: arrow
 	mkdir -p build/debug && \
 	cd build/debug && \
-	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=TRUE ../.. && \
+	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=TRUE -DBUILD_BENCHMARK=TRUE ../.. && \
 	cmake --build . --config Debug -- -j $(NUM_THREADS)
 
-test: all
+benchmark: arrow
+	mkdir -p build/release && \
+	cd build/release && \
+	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_BENCHMARK=TRUE ../.. && \
+	cmake --build . --config Release -- -j $(NUM_THREADS)
+
+test: arrow
+	mkdir -p build/release && \
+	cd build/release && \
+	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE ../.. && \
+	cmake --build . --config Release -- -j $(NUM_THREADS)
 	cd $(ROOT_DIR)/build/release/test && \
 	ctest
 

README.md

@@ -20,12 +20,12 @@ Kùzu is being actively developed at University of Waterloo as a feature-rich an
 To build from source code, Kùzu requires Cmake(>=3.11), Python 3, and a compiler that supports `C++20`.
 - Perform a full clean build without tests and benchmark:
   - `make clean && make`
-- Perform a full clean build with tests and benchmark:
+- Perform a full clean build with tests and benchmark (optional):
   - `make clean && make all`
 - Run tests (optional):
   - `make test`
 
-For development, use `make debug` or `make alldebug` to build a non-optimized debug version.
+For development, use `make debug` to build a non-optimized debug version.
 To build in parallel, pass `NUM_THREADS` as parameter, e.g., `make NUM_THREADS=8`.
 
 After build, our CLI binary `kuzu_shell` is available under the directory `build/release/tools/shell/`.

test/CMakeLists.txt

@@ -12,6 +12,7 @@ target_link_libraries(GTest::GTest INTERFACE gmock_main)
 
 enable_testing()
 add_subdirectory(test_helper)
+add_subdirectory(graph_test)
 add_subdirectory(binder)
 add_subdirectory(catalog)
 add_subdirectory(common)

test/binder/binder_error_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/binder/binder_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/catalog/catalog_test.cpp

@@ -1,5 +1,5 @@
 #include "catalog/catalog.h"
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::catalog;
 

test/copy/arrow_node_copy_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::common;
 using namespace kuzu::storage;

test/copy/copy_dates_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::common;
 using namespace kuzu::storage;

test/copy/copy_dos_style_newline_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::common;
 using namespace kuzu::storage;

test/copy/copy_fault_test.cpp

@@ -1,5 +1,5 @@
 #include "common/exception.h"
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/copy/copy_interval_test.cpp

@@ -1,6 +1,6 @@
 #include <string>
 
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::common;
 using namespace kuzu::storage;

test/copy/copy_lists_test.cpp

@@ -1,5 +1,5 @@
 #include "common/type_utils.h"
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::common;
 using namespace kuzu::storage;

test/copy/copy_test.cpp

@@ -1,5 +1,5 @@
 #include "common/csv_reader/csv_reader.h"
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::common;
 using namespace kuzu::catalog;

test/copy/copy_timestamp_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::common;
 using namespace kuzu::storage;

test/demo_db/demo_db_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using ::testing::Test;
 using namespace kuzu::testing;

test/graph_test/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_library(
+        graph_test
+        OBJECT
+        graph_test.cpp)
+
+target_include_directories(
+        graph_test
+        PUBLIC
+        ../include/
+        ../../src/include
+)
+
+target_link_libraries(graph_test PUBLIC GTest::GTest kuzu)

test/test_helper/graph_test.cpp → test/graph_test/graph_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using ::testing::Test;
 

test/include/test_helper/graph_test.h → test/include/graph_test/graph_test.h

@@ -8,7 +8,7 @@
 #include "parser/parser.h"
 #include "planner/logical_plan/logical_plan_util.h"
 #include "planner/planner.h"
-#include "test_helper.h"
+#include "test_helper/test_helper.h"
 
 using namespace kuzu::main;
 using ::testing::Test;

test/include/main_test_helper/main_test_helper.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 namespace kuzu {
 namespace testing {

test/runner/e2e_copy_transaction_test.cpp

@@ -1,7 +1,7 @@
 #include <set>
 
+#include "graph_test/graph_test.h"
 #include "processor/mapper/plan_mapper.h"
-#include "test_helper/graph_test.h"
 
 using namespace kuzu::testing;
 

test/runner/e2e_ddl_test.cpp

@@ -1,5 +1,5 @@
+#include "graph_test/graph_test.h"
 #include "processor/mapper/plan_mapper.h"
-#include "test_helper/graph_test.h"
 
 using namespace kuzu::testing;
 

test/runner/e2e_delete_create_transaction_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/runner/e2e_delete_rel_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/runner/e2e_exception_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using ::testing::Test;
 using namespace kuzu::testing;

test/runner/e2e_order_by_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/runner/e2e_read_list_test.cpp

@@ -1,6 +1,6 @@
 #include <filesystem>
 
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using ::testing::Test;
 using namespace kuzu::testing;

test/runner/e2e_read_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using ::testing::Test;
 using namespace kuzu::testing;

test/runner/e2e_set_transaction_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/runner/e2e_update_node_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/runner/e2e_update_rel_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/storage/buffer_manager_test.cpp

@@ -1,5 +1,5 @@
+#include "graph_test/graph_test.h"
 #include "storage/buffer_manager/buffer_manager.h"
-#include "test_helper/graph_test.h"
 
 using namespace kuzu::testing;
 

test/storage/disk_array_update_test.cpp

@@ -1,5 +1,5 @@
 #include "common/in_mem_overflow_buffer_utils.h"
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::storage;
 using namespace kuzu::testing;

test/storage/node_insertion_deletion_test.cpp

@@ -1,5 +1,5 @@
+#include "graph_test/graph_test.h"
 #include "storage/wal_replayer.h"
-#include "test_helper/graph_test.h"
 
 using namespace kuzu::storage;
 using namespace kuzu::testing;

test/storage/rel_insertion_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

test/storage/wal_replayer_test.cpp

@@ -1,5 +1,5 @@
+#include "graph_test/graph_test.h"
 #include "storage/wal_replayer.h"
-#include "test_helper/graph_test.h"
 
 using namespace kuzu::storage;
 using namespace kuzu::testing;

test/storage/wal_test.cpp

@@ -1,4 +1,4 @@
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 using namespace kuzu::storage;

test/test_helper/CMakeLists.txt

@@ -2,7 +2,7 @@ add_library(
         test_helper
         OBJECT
         test_helper.cpp
-        graph_test.cpp)
+)
 
 target_include_directories(
         test_helper

test/transaction/transaction_manager_test.cpp

@@ -1,5 +1,5 @@
 #include "common/exception.h"
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 #include "transaction/transaction_manager.h"
 
 using namespace kuzu::common;

test/transaction/transaction_test.cpp

@@ -1,5 +1,5 @@
 #include "common/configs.h"
-#include "test_helper/graph_test.h"
+#include "graph_test/graph_test.h"
 
 using namespace kuzu::testing;
 

tools/CMakeLists.txt

@@ -1,6 +1,6 @@
 add_subdirectory(shell)
 add_subdirectory(python_api)
-if (${BUILD_TESTS})
+if(${BUILD_BENCHMARK})
     add_subdirectory(join_order_pick)
     add_subdirectory(benchmark)
 endif()