Merge branch 'kuzudb:master' into master

.github/workflows/ci-workflow.yml

@@ -20,12 +20,19 @@ jobs:
         run: |
           pip install torch~=1.13 --extra-index-url https://download.pytorch.org/whl/cpu &&\
           pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-1.13.0+cpu.html
+      
+      - name: Ensure Node.js dependencies
+        run: npm install --include=dev
+        working-directory: tools/nodejs_api
 
       - name: Test with coverage
         run: CC=gcc CXX=g++ make lcov NUM_THREADS=32
 
       - name: Python test
-        run: CC=clang-14 CXX=clang++-14 make pytest NUM_THREADS=32
+        run: CC=gcc CXX=g++ make pytest NUM_THREADS=32
+
+      - name: Node.js test
+        run: CC=gcc CXX=g++ make nodejstest NUM_THREADS=32
 
       - name: Generate coverage report
         run: |
@@ -49,19 +56,23 @@ jobs:
           pip install torch~=1.13 --extra-index-url https://download.pytorch.org/whl/cpu &&\
           pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-1.13.0+cpu.html
 
+      - name: Ensure Node.js dependencies
+        run: npm install --include=dev
+        working-directory: tools/nodejs_api
+
       - name: Build debug
         run: CC=gcc CXX=g++ make alldebug NUM_THREADS=32
 
       - name: Run test with ASan
-        run: ctest
+        run: ctest --output-on-failure -j 10
         env:
           LD_PRELOAD: "/usr/lib/x86_64-linux-gnu/libasan.so.6"
           ASAN_OPTIONS: "detect_leaks=1:log_path=/tmp/asan.log"
         working-directory: ./build/debug/test
         continue-on-error: true
 
-      - name: Display ASan log
-        run: cat /tmp/asan.log* || true
+      - name: Report ASan log
+        run: ls /tmp/asan.log* 1>/dev/null 2>&1 && (cat /tmp/asan.log*; exit 1) || exit 0
         shell: bash
 
       - name: Clean up ASan log
@@ -79,6 +90,10 @@ jobs:
           pip install torch~=1.13 --extra-index-url https://download.pytorch.org/whl/cpu &&\
           pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-1.13.0+cpu.html
 
+      - name: Ensure Node.js dependencies
+        run: npm install --include=dev
+        working-directory: tools/nodejs_api
+
       - name: Build
         run: CC=clang-14 CXX=clang++-14 make release NUM_THREADS=32
 
@@ -87,6 +102,22 @@ jobs:
 
       - name: Python test
         run: CC=clang-14 CXX=clang++-14 make pytest NUM_THREADS=32
+
+      - name: Node.js test
+        run: CC=clang-14 CXX=clang++-14 make nodejstest NUM_THREADS=32
+
+  msvc-build-test:
+    name: msvc build & test
+    needs: [clang-formatting-check]
+    runs-on: self-hosted-windows
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build and test
+        shell: cmd
+        run: |
+          call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+          make test NUM_THREADS=18
 
   clang-formatting-check:
     name: clang-format check
@@ -111,6 +142,10 @@ jobs:
         run: |
           pip install torch~=1.13 --extra-index-url https://download.pytorch.org/whl/cpu &&\
           pip install --user -r tools/python_api/requirements_dev.txt -f https://data.pyg.org/whl/torch-1.13.0+cpu.html
+      
+      - name: Ensure Node.js dependencies
+        run: npm install
+        working-directory: tools/nodejs_api
 
       - name: Build
         run: make benchmark NUM_THREADS=30

.github/workflows/linux-nodejs-workflow.yml

@@ -0,0 +1,33 @@
+name: Build-Linux-NodeJS-Module
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-linux-nodejs:
+    runs-on: kuzu-self-hosted-linux-building
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create Node.js source distribution
+        working-directory: tools/nodejs_api
+        run: |
+          rm -rf kuzu-source.tar.gz package *.node
+          node package
+
+      - name: Extract tarball
+        working-directory: tools/nodejs_api
+        run: tar -xvzf kuzu-source.tar.gz
+
+      - name: Build Node.js native module
+        working-directory: tools/nodejs_api/package
+        run: npm i
+
+      - name: Move Node.js native module
+        working-directory: tools/nodejs_api/package
+        run: mv kuzujs.node kuzujs-linux-x64.node
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: linux-nodejs-module
+          path: tools/nodejs_api/package/kuzujs-linux-x64.node

.github/workflows/mac-nodejs-workflow.yml

@@ -0,0 +1,55 @@
+name: Build-Mac-NodeJS-Module
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-mac-nodejs:
+    runs-on: self-hosted-mac-arm
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create Node.js source distribution
+        working-directory: tools/nodejs_api
+        run: |
+          rm -rf kuzu-source.tar.gz package *.node
+          node package
+
+      - name: Extract tarball
+        working-directory: tools/nodejs_api
+        run: tar -xvzf kuzu-source.tar.gz
+
+      - name: Build Node.js native module for Apple Silicon
+        working-directory: tools/nodejs_api/package
+        run: npm i
+        env:
+          MACOSX_DEPLOYMENT_TARGET: 11.0
+          ARCHFLAGS: "-arch arm64"
+
+      - name: Move Node.js native module
+        working-directory: tools/nodejs_api/
+        run: mv package/kuzujs.node kuzujs-darwin-arm64.node
+
+      - name: Clean up
+        working-directory: tools/nodejs_api/
+        run: rm -rf package
+
+      - name: Extract tarball
+        working-directory: tools/nodejs_api
+        run: tar -xvzf kuzu-source.tar.gz
+
+      - name: Build Node.js native module for Intel
+        working-directory: tools/nodejs_api/package
+        run: npm i
+        env:
+          MACOSX_DEPLOYMENT_TARGET: 10.15
+          ARCHFLAGS: "-arch x86_64"
+
+      - name: Move Node.js native module
+        working-directory: tools/nodejs_api/
+        run: mv package/kuzujs.node kuzujs-darwin-x64.node
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: mac-nodejs-module
+          path: tools/nodejs_api/kuzujs-darwin-*.node

.github/workflows/mac-wheel-workflow.yml

@@ -21,7 +21,7 @@ jobs:
         run: |
           rm -rf wheelhouse kuzu.tar.gz
           mkdir wheelhouse
-          bash package_tar.sh
+          python3 package_tar.py
 
       - name: Build wheels for Apple Silicon
         uses: pypa/cibuildwheel@v2.12.0

.github/workflows/windows-nodejs-workflow.yml

@@ -0,0 +1,36 @@
+name: Build-Windows-NodeJS-Module
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-windows-nodejs:
+    runs-on: self-hosted-windows
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create Node.js source distribution
+        working-directory: tools/nodejs_api
+        run: |
+          node clean
+          node package
+
+      - name: Extract tarball
+        working-directory: tools/nodejs_api
+        run: tar -xvzf kuzu-source.tar.gz
+
+      - name: Build Node.js native module
+        working-directory: tools/nodejs_api/package
+        shell: cmd
+        run: |
+          call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+          npm i
+
+      - name: Move Node.js native module
+        working-directory: tools/nodejs_api/package
+        run: ren kuzujs.node kuzujs-win32-x64.node
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: windows-nodejs-module
+          path: tools/nodejs_api/package/kuzujs-win32-x64.node

.github/workflows/windows-precompiled-bin-workflow.yml

@@ -0,0 +1,42 @@
+name: Build-Windows-Precompiled-Binaries
+
+on:
+  workflow_dispatch:
+    inputs:
+      packageVersion:
+        description: "Version of output package"
+        required: true
+        default: "0.0.1"
+
+env:
+  PACKAGE_VERSION: ${{ github.event.inputs.packageVersion }}
+
+jobs:
+  build-precompiled-bin:
+    runs-on: self-hosted-windows
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install networkx
+        run: python.exe -m pip install networkx --user
+
+      - name: Build precompiled binaries
+        run: python.exe build.py
+        working-directory: ./scripts/pre-compiled-bins/
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: libkuzu-${{ github.event.inputs.packageVersion }}-windows-x86_64
+          path: |
+            ./scripts/pre-compiled-bins/kuzu.h
+            ./scripts/pre-compiled-bins/kuzu.hpp
+            ./scripts/pre-compiled-bins/kuzu_shared.dll
+            ./scripts/pre-compiled-bins/kuzu_shared.lib
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: kuzu_cli-${{ github.event.inputs.packageVersion }}-windows-x86_64
+          path: ./scripts/pre-compiled-bins/kuzu.exe
+
+      - name: Clean up
+        run: rm -rf ./scripts/pre-compiled-bins/kuzu ./scripts/pre-compiled-bins/headers ./scripts/pre-compiled-bins/kuzu_shared.dll ./scripts/pre-compiled-bins/kuzu_shared.lib ./scripts/pre-compiled-bins/kuzu.h ./scripts/pre-compiled-bins/kuzu.hpp

.github/workflows/windows-wheel-workflow.yml

@@ -0,0 +1,54 @@
+name: Build-Windows-Wheels
+
+on:
+  workflow_dispatch:
+    inputs:
+      pythonPackageVersion:
+        description: "Version of output Python package"
+        required: true
+        default: "0.0.1"
+
+env:
+  PYTHON_PACKAGE_VERSION: ${{ github.event.inputs.pythonPackageVersion }}
+
+jobs:
+  build-wheels:
+    runs-on: self-hosted-windows
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create source distribution
+        working-directory: ./scripts/pip-package/
+        shell: cmd
+        run: |
+          if exist wheelhouse rmdir /s /q wheelhouse
+          mkdir wheelhouse
+          python.exe package_tar.py
+
+      # Manually set up cibuildwheel instead of importing their
+      # action since there seems to be an issue with quoting
+      # --only when inputs.only is empty.
+      # See https://github.com/pypa/cibuildwheel/blob/main/action.yml
+      - uses: actions/setup-python@v4
+        id: python
+        with:
+          python-version: "3.7 - 3.11"
+          update-environment: false
+
+      - name: Build Wheels
+        env:
+          CIBW_ARCHS_WINDOWS: AMD64
+          CIBW_SKIP: pp* cp36*
+          CIBW_BUILD_VERBOSITY: 3
+        shell: cmd
+        working-directory: ./scripts/pip-package/
+        run:
+          call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
+
+          pipx run --spec cibuildwheel cibuildwheel
+                   "kuzu.tar.gz"
+                   --output-dir "wheelhouse"
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./scripts/pip-package/wheelhouse/*.whl

.gitignore

@@ -1,8 +1,11 @@
 .idea/
 .vscode
+.vs
 bazel-*
 .clwb
+.cache
 .bazelrc
+test/test_files/test_list
 
 test_temp/
 build/
@@ -13,7 +16,8 @@ __pycache__/
 *.py[cod]
 *$py.class
 cmake-build-debug/
-test/unittest_temp/
+test/unittest_temp_*
+dataset/parquet_temp_*
 tools/python_api/test/test_PYTHON_CSV.csv
 
 # antlr4 jar
@@ -26,7 +30,7 @@ scripts/antlr4/antlr4.jar
 tools/nodejs_api/node_modules/
 tools/nodejs_api/cmake_install.cmake
 tools/nodejs_api/package-lock.json
-tools/nodejs_api/testDb/
+tools/nodejs_api/prebuilt/
 
 # Archive files
 *.zip