Add Windows Support

.github/workflows/ci-workflow.yml

@@ -88,6 +88,19 @@ jobs:
       - name: Python test
         run: CC=clang-14 CXX=clang++-14 make pytest NUM_THREADS=32
 
+  msvc-build-test:
+    name: msvc build & test
+#     needs: [clang-formatting-check]
+    runs-on: self-hosted-windows
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build and test
+        shell: cmd
+        run: |
+          call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+          make test NUM_THREADS=24
+
   clang-formatting-check:
     name: clang-format check
     runs-on: kuzu-self-hosted-testing

CMakeLists.txt

@@ -9,6 +9,10 @@ set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Release)
+endif()
+
 if(DEFINED ENV{PYBIND11_PYTHON_VERSION})
     set(PYBIND11_PYTHON_VERSION $ENV{PYBIND11_PYTHON_VERSION})
 endif()
@@ -43,14 +47,48 @@ set(INSTALL_CMAKE_DIR
 option(ENABLE_ADDRESS_SANITIZER "Enable address sanitizer." FALSE)
 option(ENABLE_THREAD_SANITIZER "Enable thread sanitizer." FALSE)
 option(ENABLE_UBSAN "Enable undefined behavior sanitizer." FALSE)
+option(USE_SYSTEM_ARROW "Use system version of arrow" FALSE)
+if(MSVC)
+    # Required for M_PI on Windows
+    add_compile_definitions(_USE_MATH_DEFINES)
+    add_compile_definitions(NOMINMAX)
+    add_compile_definitions(ARROW_STATIC PARQUET_STATIC)
+    # TODO (bmwinger): Figure out if this can be set automatically by cmake,
+    # or at least better integrated with user-specified options
+    # For now, hardcode _AMD64_
+    # CMAKE_GENERATOR_PLATFORM can be used for visual studio builds, but not for ninja
+    add_compile_definitions(_AMD64_)
+endif()
+if(CMAKE_BUILD_TYPE MATCHES Release)
+    if(MSVC)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /O2 /utf-8")
+        string(REGEX REPLACE "/W[3|4]" "/w" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+        add_compile_options(/W0)
+    else()
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
+    endif()
+endif()
+
 if(${ENABLE_THREAD_SANITIZER})
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fno-common -fpermissive")
+    if(MSVC)
+        message(FATAL_ERROR "Thread sanitizer is not supported on MSVC")
+    else()
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fno-common -fpermissive")
+    endif()
 endif()
 if(${ENABLE_ADDRESS_SANITIZER})
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -fno-common -fpermissive")
+    if(MSVC)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /fsanitize=address")
+    else()
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer -fno-common -fpermissive")
+    endif()
 endif()
 if(${ENABLE_UBSAN})
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-common -fpermissive")
+    if(MSVC)
+        message(FATAL_ERROR "Undefined behavior sanitizer is not supported on MSVC")
+    else()
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-common -fpermissive")
+    endif()
 endif()
 option(BUILD_TESTS "Build C++ and Python tests." FALSE)
 option(BUILD_BENCHMARK "Build benchmarks." FALSE)
@@ -80,18 +118,46 @@ endfunction()
 add_definitions(-DKUZU_ROOT_DIRECTORY="${PROJECT_SOURCE_DIR}")
 add_definitions(-DKUZU_STORAGE_VERSION="${CMAKE_PROJECT_VERSION}")
 
-set(ARROW_INSTALL ${CMAKE_CURRENT_SOURCE_DIR}/external/build/arrow/install)
-find_library(ARROW_DEPS_PATH arrow_bundled_dependencies HINTS ${ARROW_INSTALL}/lib ${ARROW_INSTALL}/lib64)
-find_library(PARQUET_PATH parquet HINTS ${ARROW_INSTALL}/lib ${ARROW_INSTALL}/lib64)
-find_library(ARROW_PATH arrow HINTS ${ARROW_INSTALL}/lib ${ARROW_INSTALL}/lib64)
-
-add_library(arrow_deps STATIC IMPORTED)
-set_target_properties(arrow_deps PROPERTIES IMPORTED_LOCATION ${ARROW_DEPS_PATH})
-add_library(parquet_lib STATIC IMPORTED)
-set_target_properties(parquet_lib PROPERTIES IMPORTED_LOCATION ${PARQUET_PATH})
-add_library(arrow_lib STATIC IMPORTED)
-set_target_properties(arrow_lib PROPERTIES IMPORTED_LOCATION ${ARROW_PATH})
-include_directories(${ARROW_INSTALL}/include)
+if (${USE_SYSTEM_ARROW})
+    find_package(Arrow REQUIRED)
+    find_package(Parquet REQUIRED)
+    if (TARGET arrow_shared)
+        set(ARROW_LIB arrow_shared)
+    else()
+        set(ARROW_LIB arrow_static)
+    endif()
+    if (TARGET parquet_shared)
+        set(PARQUET_LIB parquet_shared)
+    else()
+        set(PARQUET_LIB parquet_static)
+    endif()
+else()
+    if (NOT DEFINED ARROW_INSTALL)
+        message(STATUS "Configuring arrow for bundled install")
+        set(ARROW_INSTALL ${CMAKE_CURRENT_SOURCE_DIR}/external/build/arrow/install)
+    else()
+        message(STATUS "Using arrow at path ${ARROW_INSTALL}")
+    endif()
+    find_library(ARROW_DEPS_PATH arrow_bundled_dependencies HINTS ${ARROW_INSTALL}/lib ${ARROW_INSTALL}/lib64)
+    if(WIN32)
+        find_library(PARQUET_PATH parquet_static HINTS ${ARROW_INSTALL}/lib ${ARROW_INSTALL}/lib64)
+        find_library(ARROW_PATH arrow_static HINTS ${ARROW_INSTALL}/lib ${ARROW_INSTALL}/lib64)
+    else()
+        find_library(PARQUET_PATH parquet HINTS ${ARROW_INSTALL}/lib ${ARROW_INSTALL}/lib64)
+        find_library(ARROW_PATH arrow HINTS ${ARROW_INSTALL}/lib ${ARROW_INSTALL}/lib64)
+    endif()
+
+    add_library(arrow_deps STATIC IMPORTED)
+    set_target_properties(arrow_deps PROPERTIES IMPORTED_LOCATION ${ARROW_DEPS_PATH})
+    add_library(parquet_lib STATIC IMPORTED)
+    set_target_properties(parquet_lib PROPERTIES IMPORTED_LOCATION ${PARQUET_PATH})
+    add_library(arrow_lib STATIC IMPORTED)
+    set_target_properties(arrow_lib PROPERTIES IMPORTED_LOCATION ${ARROW_PATH})
+    include_directories(${ARROW_INSTALL}/include)
+
+    set(ARROW_LIB arrow_lib arrow_deps)
+    set(PARQUET_LIB parquet_lib)
+endif()
 
 include_directories(src/include)
 include_directories(third_party/antlr4_cypher/include)

Makefile

@@ -4,12 +4,20 @@ GENERATOR=
 FORCE_COLOR=
 NUM_THREADS=
 SANITIZER_FLAG=
-ROOT_DIR=$(PWD)
+ROOT_DIR=$(CURDIR)
 
 ifndef $(NUM_THREADS)
 	NUM_THREADS=1
 endif
 
+ifeq ($(OS),Windows_NT)
+	ifndef $(GEN)
+		GEN=ninja
+	endif
+	SHELL := cmd.exe
+	.SHELLFLAGS := /c
+endif
+
 ifeq ($(GEN),ninja)
 	GENERATOR=-G "Ninja"
 	FORCE_COLOR=-DFORCE_COLORED_OUTPUT=1
@@ -25,54 +33,55 @@ ifeq ($(UBSAN), 1)
 	SANITIZER_FLAG=-DENABLE_ADDRESS_SANITIZER=FALSE -DENABLE_THREAD_SANITIZER=TRUE -DENABLE_UBSAN=TRUE
 endif
 
+ifeq ($(OS),Windows_NT)
+define mkdirp
+	(if not exist "$(1)" mkdir "$(1)")
+endef
+else
+define mkdirp
+	mkdir -p $(1)
+endef
+endif
+
 arrow:
-	cd external && \
-	mkdir -p build && \
-	cd build && \
-	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release ../ && \
+	$(call mkdirp,external/build) && cd external/build && \
+	cmake $(FORCE_COLOR) $(SANITIZER_FLAG) $(GENERATOR) -DCMAKE_BUILD_TYPE=Release .. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 
 release: arrow
-	mkdir -p build/release && \
-	cd build/release && \
+	$(call mkdirp,build/release) && cd build/release && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release ../.. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 
 debug: arrow
-	mkdir -p build/debug && \
-	cd build/debug && \
+	$(call mkdirp,build/debug) && cd build/debug && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Debug ../.. && \
 	cmake --build . --config Debug -- -j $(NUM_THREADS)
 
 all: arrow
-	mkdir -p build/release && \
-	cd build/release && \
+	$(call mkdirp,build/release) && cd build/release && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE -DBUILD_BENCHMARK=TRUE ../.. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 
 alldebug: arrow
-	mkdir -p build/debug && \
-	cd build/debug && \
+	$(call mkdirp,build/debug) && cd build/debug && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=TRUE -DBUILD_BENCHMARK=TRUE ../.. && \
 	cmake --build . --config Debug -- -j $(NUM_THREADS)
 
 benchmark: arrow
-	mkdir -p build/release && \
-	cd build/release && \
+	$(call mkdirp,build/release) && cd build/release && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_BENCHMARK=TRUE ../.. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 
 test: arrow
-	mkdir -p build/release && \
-	cd build/release && \
+	$(call mkdirp,build/release) && cd build/release && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE ../.. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 	cd $(ROOT_DIR)/build/release/test && \
 	ctest
 
 lcov: arrow
-	mkdir -p build/release && \
-	cd build/release && \
+	$(call mkdirp,build/release) && cd build/release && \
 	cmake $(GENERATOR) $(FORCE_COLOR) $(SANITIZER_FLAG) -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=TRUE -DBUILD_LCOV=TRUE ../.. && \
 	cmake --build . --config Release -- -j $(NUM_THREADS)
 	cd $(ROOT_DIR)/build/release/test && \
@@ -84,12 +93,24 @@ pytest: arrow
 	python3 -m pytest -v test_main.py
 
 clean-python-api:
+ifeq ($(OS),Windows_NT)
+	rmdir /s /q tools\python_api\build
+else
 	rm -rf tools/python_api/build
+endif
 
 clean-external:
+ifeq ($(OS),Windows_NT)
+	rmdir /s /q external\build
+else
 	rm -rf external/build
+endif
 
 clean: clean-python-api
+ifeq ($(OS),Windows_NT)
+	rmdir /s /q build
+else
 	rm -rf build
+endif
 
 clean-all: clean-external clean

external/arrow/apache_arrow.cmake

@@ -2,13 +2,19 @@ include(ExternalProject)
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
+if(WIN32)
+        set(ARROW_BUILD_TYPE ${CMAKE_BUILD_TYPE})
+else()
+        set(ARROW_BUILD_TYPE Release)
+endif()
+
 ExternalProject_Add(apache_arrow
         GIT_REPOSITORY "https://github.com/apache/arrow"
         GIT_TAG f10f5cfd1376fb0e602334588b3f3624d41dee7d
         PREFIX "${CMAKE_BINARY_DIR}/arrow/"
         INSTALL_DIR "${CMAKE_BINARY_DIR}/arrow/install"
         CONFIGURE_COMMAND
-        ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -DCMAKE_BUILD_TYPE=Release
+        ${CMAKE_COMMAND} -G${CMAKE_GENERATOR} -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE}
         -DBUILD_WARNING_LEVEL=PRODUCTION -DARROW_DEPENDENCY_SOURCE=BUNDLED
         -DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}
         -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}

src/CMakeLists.txt

@@ -14,12 +14,16 @@ add_subdirectory(transaction)
 
 add_library(kuzu STATIC ${ALL_OBJECT_FILES})
 target_link_libraries(kuzu
-        PUBLIC antlr4_cypher antlr4_runtime utf8proc re2 parquet_lib arrow_lib arrow_deps Threads::Threads)
+        PUBLIC antlr4_cypher antlr4_runtime utf8proc re2 ${PARQUET_LIB} ${ARROW_LIB} Threads::Threads)
 target_include_directories(kuzu
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)
 add_library(kuzu_shared SHARED ${ALL_OBJECT_FILES})
-set_target_properties(kuzu_shared PROPERTIES OUTPUT_NAME kuzu)
+if(WIN32)
+        set_target_properties(kuzu_shared PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON)
+else()
+        set_target_properties(kuzu_shared PROPERTIES OUTPUT_NAME kuzu)
+endif()
 target_link_libraries(kuzu_shared
-        PUBLIC antlr4_cypher antlr4_runtime utf8proc re2 parquet_lib arrow_lib arrow_deps Threads::Threads)
+        PUBLIC antlr4_cypher antlr4_runtime utf8proc re2 ${PARQUET_LIB} ${ARROW_LIB} Threads::Threads)
 target_include_directories(kuzu_shared
         PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>)

src/binder/bind/bind_graph_pattern.cpp

@@ -134,8 +134,8 @@ void Binder::bindQueryRel(const RelPattern& relPattern,
     const std::shared_ptr<NodeExpression>& rightNode, QueryGraph& queryGraph,
     PropertyKeyValCollection& collection) {
     auto parsedName = relPattern.getVariableName();
-    if (variablesInScope.contains(parsedName)) {
-        auto prevVariable = variablesInScope.at(parsedName);
+    if (variableScope->contains(parsedName)) {
+        auto prevVariable = variableScope->getExpression(parsedName);
         ExpressionBinder::validateExpectedDataType(*prevVariable, LogicalTypeID::REL);
         throw BinderException("Bind relationship " + parsedName +
                               " to relationship with same name is not supported.");
@@ -203,7 +203,7 @@ void Binder::bindQueryRel(const RelPattern& relPattern,
         }
     }
     if (!parsedName.empty()) {
-        variablesInScope.insert({parsedName, queryRel});
+        variableScope->addExpression(parsedName, queryRel);
     }
     for (auto i = 0u; i < relPattern.getNumPropertyKeyValPairs(); ++i) {
         auto [propertyName, rhs] = relPattern.getProperty(i);
@@ -235,8 +235,8 @@ std::shared_ptr<NodeExpression> Binder::bindQueryNode(
     const NodePattern& nodePattern, QueryGraph& queryGraph, PropertyKeyValCollection& collection) {
     auto parsedName = nodePattern.getVariableName();
     std::shared_ptr<NodeExpression> queryNode;
-    if (variablesInScope.contains(parsedName)) { // bind to node in scope
-        auto prevVariable = variablesInScope.at(parsedName);
+    if (variableScope->contains(parsedName)) { // bind to node in scope
+        auto prevVariable = variableScope->getExpression(parsedName);
         ExpressionBinder::validateExpectedDataType(*prevVariable, LogicalTypeID::NODE);
         queryNode = static_pointer_cast<NodeExpression>(prevVariable);
         // E.g. MATCH (a:person) MATCH (a:organisation)
@@ -281,7 +281,7 @@ std::shared_ptr<NodeExpression> Binder::createQueryNode(const NodePattern& nodeP
         queryNode->addPropertyExpression(propertyName, std::move(propertyExpression));
     }
     if (!parsedName.empty()) {
-        variablesInScope.insert({parsedName, queryNode});
+        variableScope->addExpression(parsedName, queryNode);
     }
     return queryNode;
 }

src/binder/bind/bind_projection_clause.cpp

@@ -16,7 +16,7 @@ std::unique_ptr<BoundWithClause> Binder::bindWithClause(const WithClause& withCl
         projectionBody->getIsDistinct(), std::move(boundProjectionExpressions));
     bindOrderBySkipLimitIfNecessary(*boundProjectionBody, *projectionBody);
     validateOrderByFollowedBySkipOrLimitInWithClause(*boundProjectionBody);
-    variablesInScope.clear();
+    variableScope->clear();
     addExpressionsToScope(boundProjectionBody->getProjectionExpressions());
     auto boundWithClause = std::make_unique<BoundWithClause>(std::move(boundProjectionBody));
     if (withClause.hasWhereExpression()) {
@@ -54,11 +54,11 @@ expression_vector Binder::bindProjectionExpressions(
         boundProjectionExpressions.push_back(expressionBinder.bindExpression(*expression));
     }
     if (containsStar) {
-        if (variablesInScope.empty()) {
+        if (variableScope->empty()) {
             throw BinderException(
                 "RETURN or WITH * is not allowed when there are no variables in scope.");
         }
-        for (auto& [name, expression] : variablesInScope) {
+        for (auto& expression : variableScope->getExpressions()) {
             boundProjectionExpressions.push_back(expression);
         }
     }
@@ -166,7 +166,7 @@ void Binder::addExpressionsToScope(const expression_vector& projectionExpression
     for (auto& expression : projectionExpressions) {
         // In RETURN clause, if expression is not aliased, its input name will serve its alias.
         auto alias = expression->hasAlias() ? expression->getAlias() : expression->toString();
-        variablesInScope.insert({alias, expression});
+        variableScope->addExpression(alias, expression);
     }
 }
 

src/binder/bind/bind_query.cpp

@@ -10,7 +10,7 @@ std::unique_ptr<BoundRegularQuery> Binder::bindQuery(const RegularQuery& regular
     for (auto i = 0u; i < regularQuery.getNumSingleQueries(); i++) {
         // Don't clear scope within bindSingleQuery() yet because it is also used for subquery
         // binding.
-        variablesInScope.clear();
+        variableScope->clear();
         boundSingleQueries.push_back(bindSingleQuery(*regularQuery.getSingleQuery(i)));
     }
     validateUnionColumnsOfTheSameType(boundSingleQueries);

src/binder/bind/bind_reading_clause.cpp

@@ -23,7 +23,6 @@ std::unique_ptr<BoundReadingClause> Binder::bindReadingClause(const ReadingClaus
 
 std::unique_ptr<BoundReadingClause> Binder::bindMatchClause(const ReadingClause& readingClause) {
     auto& matchClause = (MatchClause&)readingClause;
-    auto prevVariablesInScope = variablesInScope;
     auto [queryGraphCollection, propertyCollection] =
         bindGraphPattern(matchClause.getPatternElements());
     auto boundMatchClause =