CI: Add nightly build pipeline (#2378)

* Add nightly build pipeline

* Update builder image

* Fix update nightly version

* Typo fix

* Update CMake version to 0.0.13.1.

* Commit changes via git for archiving

* Config name for git

* Rename jar archive

* Add a dot to rust version

.github/workflows/build-and-deploy.yml

@@ -1,5 +1,8 @@
 name: Build-And-Deploy
 on:
+  schedule:
+    - cron: "0 8 * * *"
+
   workflow_dispatch:
     inputs:
       isDeploy:
@@ -32,41 +35,46 @@ on:
         type: boolean
         required: true
         default: false
+      isNightly:
+        description: "Whether the build is a nightly build?"
+        type: boolean
+        required: true
+        default: false
 
 jobs:
   build-java-mac:
-    if: ${{ github.event.inputs.skipJava != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipJava != 'true' }}
     uses: ./.github/workflows/mac-java-workflow.yml
     secrets: inherit
 
   build-java-linux:
-    if: ${{ github.event.inputs.skipJava != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipJava != 'true' }}
     uses: ./.github/workflows/linux-java-workflow.yml
     secrets: inherit
 
   build-java-windows:
-    if: ${{ github.event.inputs.skipJava != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipJava != 'true' }}
     uses: ./.github/workflows/windows-java-workflow.yml
     secrets: inherit
 
   inject-java-bins:
-    if: ${{ github.event.inputs.skipJava != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipJava != 'true' }}
     needs: [build-java-mac, build-java-linux, build-java-windows]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v3
         with:
-          name: libkuzu-java-osx-x86_64
+          name: java-lib-osx-x86_64
           path: java-bins
 
       - uses: actions/download-artifact@v3
         with:
-          name: libkuzu-java-osx-arm64
+          name: java-lib-osx-arm64
           path: java-bins
 
       - uses: actions/download-artifact@v3
         with:
-          name: libkuzu-java-linux-aarch64
+          name: java-lib-linux-aarch64
           path: java-bins
 
       - uses: actions/download-artifact@v3
@@ -76,7 +84,7 @@ jobs:
 
       - uses: actions/download-artifact@v3
         with:
-          name: libkuzu-java-win-x86_64
+          name: java-lib-win-x86_64
           path: java-bins
 
       - name: Add Java libs to jar
@@ -90,33 +98,38 @@ jobs:
       - name: Upload jar
         uses: actions/upload-artifact@v3
         with:
-          name: kuzu-deploy-jar
+          name: kuzu-java-multiplatform-jar
           path: java-bins/kuzu_java.jar
 
   build-nodejs-mac:
-    if: ${{ github.event.inputs.skipNodejs != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipNodejs != 'true' }}
     uses: ./.github/workflows/mac-nodejs-workflow.yml
     secrets: inherit
 
   build-nodejs-linux:
-    if: ${{ github.event.inputs.skipNodejs != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipNodejs != 'true' }}
     uses: ./.github/workflows/linux-nodejs-workflow.yml
     secrets: inherit
 
   build-nodejs-windows:
-    if: ${{ github.event.inputs.skipNodejs != 'true' }}
+    if: ${{ github.event_name == 'schedule' ||  github.event.inputs.skipNodejs != 'true' }}
     uses: ./.github/workflows/windows-nodejs-workflow.yml
     secrets: inherit
 
   deploy-nodejs:
-    if: ${{ github.event.inputs.skipNodejs != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipNodejs != 'true' }}
     needs: [build-nodejs-mac, build-nodejs-linux, build-nodejs-windows]
     runs-on: ubuntu-latest
     env:
       NODE_AUTH_TOKEN: ${{ secrets.NPM_JS_TOKEN }}
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update nightly version
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.isNightly == 'true' }}
+        run: python3 update-nightly-build-version.py
+        working-directory: scripts
+
       - name: Create prebuilt folder
         run: mkdir -p tools/nodejs_api/prebuilt
 
@@ -165,36 +178,47 @@ jobs:
           path: tools/nodejs_api/kuzu-source.tar.gz
 
       - name: Deploy to npm.js dry run
-        if: ${{ github.event.inputs.isDeploy != 'true' }}
+        if: ${{ github.event_name != 'schedule' && github.event.inputs.isDeploy != 'true' }}
         run: npm publish kuzu-source.tar.gz --access public --dry-run
         working-directory: tools/nodejs_api
 
       - name: Deploy to npm.js
-        if: ${{ github.event.inputs.isDeploy == 'true' }}
+        if: ${{ github.event_name == 'schedule' ||  github.event.inputs.isDeploy == 'true' }}
         run: npm publish kuzu-source.tar.gz --access public
         working-directory: tools/nodejs_api
 
   build-wheel-mac:
-    if: ${{ github.event.inputs.skipPython != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipPython != 'true' }}
     uses: ./.github/workflows/mac-wheel-workflow.yml
+    with:
+      isNightly: ${{ github.event_name == 'schedule' || github.event.inputs.isNightly == 'true' }}
     secrets: inherit
 
   build-wheel-linux:
-    if: ${{ github.event.inputs.skipPython != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipPython != 'true' }}
     uses: ./.github/workflows/linux-wheel-workflow.yml
+    with:
+      isNightly: ${{ github.event_name == 'schedule' || github.event.inputs.isNightly == 'true' }}
     secrets: inherit
 
   build-wheel-windows:
-    if: ${{ github.event.inputs.skipPython != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipPython != 'true' }}
     uses: ./.github/workflows/windows-wheel-workflow.yml
+    with:
+      isNightly: ${{ github.event_name == 'schedule' || github.event.inputs.isNightly == 'true' }}
     secrets: inherit
 
   package-python-sdist:
-    if: ${{ github.event.inputs.skipPython != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipPython != 'true' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update nightly version
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.isNightly == 'true' }}
+        run: python3 update-nightly-build-version.py
+        working-directory: scripts
+
       - name: Package Python sdist
         run: python package_tar.py
         working-directory: scripts/pip-package
@@ -206,7 +230,7 @@ jobs:
           path: scripts/pip-package/*.tar.gz
 
   deploy-python:
-    if: ${{ github.event.inputs.skipPython != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipPython != 'true' }}
     needs:
       [
         build-wheel-mac,
@@ -257,38 +281,43 @@ jobs:
           path: dist/*
 
       - name: Deploy to PyPI test
-        if: ${{ github.event.inputs.isDeploy != 'true' }}
+        if: ${{ github.event_name != 'schedule' && github.event.inputs.isDeploy != 'true' }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_TEST_TOKEN }}
           repository-url: https://test.pypi.org/legacy/
 
       - name: Deploy to PyPI
-        if: ${{ github.event.inputs.isDeploy == 'true' }}
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.isDeploy == 'true' }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_TOKEN }}
 
   deploy-rust:
-    if: ${{ github.event.inputs.skipRust != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipRust != 'true' }}
     runs-on: kuzu-self-hosted-testing
     env:
       CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update nightly version
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.isNightly == 'true' }}
+        run: python3 update-nightly-build-version.py
+        working-directory: scripts
+
       - name: Update Cargo.toml version
         run: python3 update_version.py
         working-directory: tools/rust_api
 
       - name: Deploy crate to Crates.io
         run: cargo publish --allow-dirty
-        if: ${{ github.event.inputs.isDeploy == 'true' }}
+        if: ${{ github.event_name == 'schedule' || github.event.inputs.isDeploy == 'true' }}
         working-directory: tools/rust_api
 
       - name: Test publishing crate
         run: cargo publish --dry-run --allow-dirty
-        if: ${{ github.event.inputs.isDeploy != 'true' }}
+        if: ${{ github.event_name != 'schedule' && github.event.inputs.isDeploy != 'true' }}
         working-directory: tools/rust_api
 
       - name: Upload crate
@@ -298,16 +327,16 @@ jobs:
           path: tools/rust_api/target/package/*.crate
 
   build-precompiled-bin-mac:
-    if: ${{ github.event.inputs.skipBinaries != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipBinaries != 'true' }}
     uses: ./.github/workflows/mac-precompiled-bin-workflow.yml
     secrets: inherit
 
   build-precompiled-bin-linux:
-    if: ${{ github.event.inputs.skipBinaries != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipBinaries != 'true' }}
     uses: ./.github/workflows/linux-precompiled-bin-workflow.yml
     secrets: inherit
 
   build-precompiled-bin-windows:
-    if: ${{ github.event.inputs.skipBinaries != 'true' }}
+    if: ${{ github.event_name == 'schedule' || github.event.inputs.skipBinaries != 'true' }}
     uses: ./.github/workflows/windows-precompiled-bin-workflow.yml
     secrets: inherit

.github/workflows/linux-java-workflow.yml

@@ -14,7 +14,7 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: libkuzu-java-linux-x86_64
+          name: java-lib-linux-x86_64
           path: tools/java_api/build/libkuzu_java_native*
 
       - uses: actions/upload-artifact@v3
@@ -32,5 +32,5 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: libkuzu-java-linux-aarch64
+          name: java-lib-linux-aarch64
           path: tools/java_api/build/libkuzu_java_native*

.github/workflows/linux-wheel-workflow.yml

@@ -3,18 +3,28 @@ name: Build-Linux-Wheels
 on:
   workflow_dispatch:
   workflow_call:
+    inputs:
+      isNightly:
+        type: boolean
+        required: true
+        default: false
 
 jobs:
   build-linux-wheels-x86_64:
     runs-on: kuzu-self-hosted-linux-building
     steps:
       - uses: actions/checkout@v3
 
-      - name: Build Wheels
+      - name: Update nightly version
+        if: ${{ github.event.inputs.isNightly == 'true' }}
+        run: /opt/python/cp310-cp310/bin/python update-nightly-build-version.py
+        working-directory: scripts
+
+      - name: Build wheels
         working-directory: ./scripts/pip-package/
         run: |
-           cp Dockerfiles/x86_64/build_all_packages.sh .
-           bash build_all_packages.sh
+          cp Dockerfiles/x86_64/build_all_packages.sh .
+          bash build_all_packages.sh
 
       - uses: actions/upload-artifact@v3
         with:
@@ -26,11 +36,16 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Build Wheels
+      - name: Update nightly version
+        if: ${{ github.event.inputs.isNightly == 'true' }}
+        run: /opt/python/cp310-cp310/bin/python update-nightly-build-version.py
+        working-directory: scripts
+
+      - name: Build wheels
         working-directory: ./scripts/pip-package/
         run: |
-           cp Dockerfiles/aarch64/build_all_packages.sh .
-           bash build_all_packages.sh
+          cp Dockerfiles/aarch64/build_all_packages.sh .
+          bash build_all_packages.sh
 
       - uses: actions/upload-artifact@v3
         with:

.github/workflows/mac-java-workflow.yml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: libkuzu-java-osx-arm64
+          name: java-lib-osx-arm64
           path: tools/java_api/build/libkuzu_java_native*
 
   build-mac-java-x86:
@@ -35,5 +35,5 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: libkuzu-java-osx-x86_64
+          name: java-lib-osx-x86_64
           path: tools/java_api/build/libkuzu_java_native*

.github/workflows/mac-wheel-workflow.yml

@@ -3,13 +3,23 @@ name: Build-Mac-Wheels
 on:
   workflow_dispatch:
   workflow_call:
+    inputs:
+      isNightly:
+        type: boolean
+        required: true
+        default: false
 
 jobs:
   build-wheels-arm64:
     runs-on: self-hosted-mac-arm
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update nightly version
+        if: ${{ github.event.inputs.isNightly == 'true' }}
+        run: python3 update-nightly-build-version.py
+        working-directory: scripts
+
       - name: Create source distribution
         working-directory: ./scripts/pip-package/
         run: |
@@ -38,6 +48,11 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Update nightly version
+        if: ${{ github.event.inputs.isNightly == 'true' }}
+        run: python3 update-nightly-build-version.py
+        working-directory: scripts
+
       - name: Create source distribution
         working-directory: ./scripts/pip-package/
         run: |

.github/workflows/windows-java-workflow.yml

@@ -18,5 +18,5 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: libkuzu-java-win-x86_64
+          name: java-lib-win-x86_64
           path: tools/java_api/build/libkuzu_java_native*