Add CI pipeline for build and deploy everything (#1796)

* X * X * Node.js publish attempt * Try to fix Windows Node.js workflow * Use backslash for Windows path * Remove path * Typo fix * Increase Windows Node.js build threads * Add missing working dir * Setup global env * Fix JAR path * Fix version number * Also upload Node.js deploy archive * Auto detect version for python API * Create source dist with version number * Test sdist * Add deploy * Remove version from bin workflows * Remove unused env * OK * Remove -sdist * Add precompiled bin workflow * Try to fix Windows CI precompile * Remove cleanup * Format yaml * Reorder * Minor fix
kuzudb · Jul 10, 2023 · 81fef9b · 81fef9b
1 parent 3a76441
commit 81fef9b
Show file tree

Hide file tree

Showing 14 changed files with 221 additions and 94 deletions.
diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml
@@ -27,33 +27,192 @@ jobs:
     steps:
       - uses: actions/download-artifact@v3
         with:
-          name: libkuzu-java-win-x86_64
+          name: libkuzu-java-osx-x86_64
           path: java-bins
 
       - uses: actions/download-artifact@v3
         with:
-          name: libkuzu-java-osx-x86_64
+          name: libkuzu-java-osx-arm64
           path: java-bins
 
       - uses: actions/download-artifact@v3
         with:
-          name: libkuzu-java-osx-arm64
+          name: kuzu-linux-jar
           path: java-bins
 
       - uses: actions/download-artifact@v3
         with:
-          name: kuzu-linux-jar
+          name: libkuzu-java-win-x86_64
           path: java-bins
 
       - name: Add Java libs to jar
         run: |
-          ls -alR java-bins
-          jar uf java-bins/kuzu-linux-jar/kuzu_java.jar java-bins/libkuzu-java-win-x86_64/libkuzu_java_native.so_windows_amd64
-          jar uf java-bins/kuzu-linux-jar/kuzu_java.jar java-bins/libkuzu-java-osx-x86_64/libkuzu_java_native.so_osx_amd64
-          jar uf java-bins/kuzu-linux-jar/kuzu_java.jar java-bins/libkuzu-java-osx-arm64/libkuzu_java_native.so_osx_arm64
-      
+          jar uf kuzu_java.jar libkuzu_java_native.so_windows_amd64
+          jar uf kuzu_java.jar libkuzu_java_native.so_osx_arm64
+          jar uf kuzu_java.jar libkuzu_java_native.so_osx_amd64
+        working-directory: java-bins
+
       - name: Upload jar
         uses: actions/upload-artifact@v3
         with:
           name: kuzu-deploy-jar
-          path: java-bins/kuzu-linux-jar/kuzu_java.jar
+          path: java-bins/kuzu_java.jar
+
+  build-nodejs-mac:
+    uses: ./.github/workflows/mac-nodejs-workflow.yml
+    secrets: inherit
+
+  build-nodejs-linux:
+    uses: ./.github/workflows/linux-nodejs-workflow.yml
+    secrets: inherit
+
+  build-nodejs-windows:
+    uses: ./.github/workflows/windows-nodejs-workflow.yml
+    secrets: inherit
+
+  deploy-nodejs:
+    needs: [build-nodejs-mac, build-nodejs-linux, build-nodejs-windows]
+    runs-on: ubuntu-latest
+    env:
+      NODE_AUTH_TOKEN: ${{ secrets.NPM_JS_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create prebuilt folder
+        run: mkdir -p tools/nodejs_api/prebuilt
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: mac-nodejs-module
+          path: tools/nodejs_api/prebuilt
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: linux-nodejs-module
+          path: tools/nodejs_api/prebuilt
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-nodejs-module
+          path: tools/nodejs_api/prebuilt
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "16"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Package Node.js API with prebuilt binaries
+        run: node package
+        working-directory: tools/nodejs_api
+
+      - name: Show tarball contents
+        run: tar -tvf kuzu-source.tar.gz
+        working-directory: tools/nodejs_api
+
+      - name: Upload tarball
+        uses: actions/upload-artifact@v3
+        with:
+          name: kuzu-deploy-nodejs
+          path: tools/nodejs_api/kuzu-source.tar.gz
+
+      - name: Deploy to npm.js dry run
+        if: ${{ github.event.inputs.isDeploy != 'true' }}
+        run: npm publish kuzu-source.tar.gz --access public --dry-run
+        working-directory: tools/nodejs_api
+
+      - name: Deploy to npm.js
+        if: ${{ github.event.inputs.isDeploy == 'true' }}
+        run: npm publish kuzu-source.tar.gz --access public
+        working-directory: tools/nodejs_api
+
+  build-wheel-mac:
+    uses: ./.github/workflows/mac-wheel-workflow.yml
+    secrets: inherit
+
+  build-wheel-linux:
+    uses: ./.github/workflows/linux-wheel-workflow.yml
+    secrets: inherit
+
+  build-wheel-windows:
+    uses: ./.github/workflows/windows-wheel-workflow.yml
+    secrets: inherit
+
+  package-python-sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Package Python sdist
+        run: python package_tar.py
+        working-directory: scripts/pip-package
+
+      - name: Upload tarball
+        uses: actions/upload-artifact@v3
+        with:
+          name: python-sdist
+          path: scripts/pip-package/*.tar.gz
+
+  deploy-python:
+    needs:
+      [
+        build-wheel-mac,
+        build-wheel-linux,
+        build-wheel-windows,
+        package-python-sdist,
+      ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: macos-wheels
+          path: dist
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: linux-wheels
+          path: dist
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-wheels
+          path: dist
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: python-sdist
+          path: dist
+
+      - name: List wheels
+        run: ls -l
+        working-directory: dist
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: kuzu-deploy-wheels
+          path: dist/*
+
+      - name: Deploy to PyPI test
+        if: ${{ github.event.inputs.isDeploy != 'true' }}
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TEST_TOKEN }}
+          repository-url: https://test.pypi.org/legacy/
+
+      - name: Deploy to PyPI
+        if: ${{ github.event.inputs.isDeploy == 'true' }}
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}
+
+  build-precompiled-bin-mac:
+    uses: ./.github/workflows/mac-precompiled-bin-workflow.yml
+    secrets: inherit
+
+  build-precompiled-bin-linux:
+    uses: ./.github/workflows/linux-precompiled-bin-workflow.yml
+    secrets: inherit
+
+  build-precompiled-bin-windows:
+    uses: ./.github/workflows/windows-precompiled-bin-workflow.yml
+    secrets: inherit
diff --git a/.github/workflows/linux-nodejs-workflow.yml b/.github/workflows/linux-nodejs-workflow.yml
@@ -2,6 +2,7 @@ name: Build-Linux-NodeJS-Module
 
 on:
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   build-linux-nodejs:

diff --git a/.github/workflows/linux-precompiled-bin-workflow.yml b/.github/workflows/linux-precompiled-bin-workflow.yml
@@ -2,14 +2,7 @@ name: Build-Linux-Precompiled-Binaries
 
 on:
   workflow_dispatch:
-    inputs:
-      packageVersion:
-        description: "Version of output package"
-        required: true
-        default: "0.0.1"
-
-env:
-  PACKAGE_VERSION: ${{ github.event.inputs.packageVersion }}
+  workflow_call:
 
 jobs:
   build-precompiled-bin:
@@ -26,16 +19,13 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: libkuzu-${{ github.event.inputs.packageVersion }}-linux-x86_64
+          name: libkuzu-linux-x86_64
           path: |
             ./scripts/pre-compiled-bins/kuzu.h
             ./scripts/pre-compiled-bins/kuzu.hpp
             ./scripts/pre-compiled-bins/libkuzu.so
 
       - uses: actions/upload-artifact@v3
         with:
-          name: kuzu_cli-${{ github.event.inputs.packageVersion }}-linux-x86_64
+          name: kuzu_cli-linux-x86_64
           path: ./scripts/pre-compiled-bins/kuzu
-
-      - name: Clean up
-        run: rm -rf ./scripts/pre-compiled-bins/kuzu ./scripts/pre-compiled-bins/headers ./scripts/pre-compiled-bins/libkuzu.so /scripts/pre-compiled-bins/kuzu.h /scripts/pre-compiled-bins/kuzu.hpp
diff --git a/.github/workflows/linux-wheel-workflow.yml b/.github/workflows/linux-wheel-workflow.yml
@@ -2,14 +2,7 @@ name: Build-Linux-Wheels
 
 on:
   workflow_dispatch:
-    inputs:
-      pythonPackageVersion:
-        description: "Version of output Python package"
-        required: true
-        default: "0.0.1"
-
-env:
-  PYTHON_PACKAGE_VERSION: ${{ github.event.inputs.pythonPackageVersion }}
+  workflow_call:
 
 jobs:
   build-linux-wheels:

diff --git a/.github/workflows/mac-nodejs-workflow.yml b/.github/workflows/mac-nodejs-workflow.yml
@@ -2,6 +2,7 @@ name: Build-Mac-NodeJS-Module
 
 on:
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   build-mac-nodejs:

diff --git a/.github/workflows/mac-precompiled-bin-workflow.yml b/.github/workflows/mac-precompiled-bin-workflow.yml
@@ -2,14 +2,7 @@ name: Build-Mac-Precompiled-Binaries
 
 on:
   workflow_dispatch:
-    inputs:
-      packageVersion:
-        description: "Version of output package"
-        required: true
-        default: "0.0.1"
-
-env:
-  PACKAGE_VERSION: ${{ github.event.inputs.packageVersion }}
+  workflow_call:
 
 jobs:
   build-precompiled-bin:
@@ -29,15 +22,15 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: libkuzu-${{ github.event.inputs.packageVersion }}-osx-arm64
+          name: libkuzu-osx-arm64
           path: |
             ./scripts/pre-compiled-bins/kuzu.h
             ./scripts/pre-compiled-bins/kuzu.hpp
             ./scripts/pre-compiled-bins/libkuzu.dylib
 
       - uses: actions/upload-artifact@v3
         with:
-          name: kuzu_cli-${{ github.event.inputs.packageVersion }}-osx-arm64
+          name: kuzu_cli-osx-arm64
           path: ./scripts/pre-compiled-bins/kuzu
 
       - name: Build precompiled binaries for Intel
@@ -49,16 +42,13 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: libkuzu-${{ github.event.inputs.packageVersion }}-osx-x86_64
+          name: libkuzu-osx-x86_64
           path: |
             ./scripts/pre-compiled-bins/kuzu.h
             ./scripts/pre-compiled-bins/kuzu.hpp
             ./scripts/pre-compiled-bins/libkuzu.dylib
 
       - uses: actions/upload-artifact@v3
         with:
-          name: kuzu_cli-${{ github.event.inputs.packageVersion }}-osx-x86_64
+          name: kuzu_cli-osx-x86_64
           path: ./scripts/pre-compiled-bins/kuzu
-
-      - name: Clean up
-        run: rm -rf ./scripts/pre-compiled-bins/kuzu ./scripts/pre-compiled-bins/headers ./scripts/pre-compiled-bins/libkuzu.dylib ./scripts/pre-compiled-bins/kuzu.h ./scripts/pre-compiled-bins/kuzu.hpp
diff --git a/.github/workflows/mac-wheel-workflow.yml b/.github/workflows/mac-wheel-workflow.yml
@@ -2,26 +2,20 @@ name: Build-Mac-Wheels
 
 on:
   workflow_dispatch:
-    inputs:
-      pythonPackageVersion:
-        description: "Version of output Python package"
-        required: true
-        default: "0.0.1"
-
-env:
-  PYTHON_PACKAGE_VERSION: ${{ github.event.inputs.pythonPackageVersion }}
+  workflow_call:
 
 jobs:
   build-wheels:
     runs-on: self-hosted-mac-arm
     steps:
       - uses: actions/checkout@v3
+
       - name: Create source distribution
         working-directory: ./scripts/pip-package/
         run: |
           rm -rf wheelhouse kuzu.tar.gz
           mkdir wheelhouse
-          python3 package_tar.py
+          python3 package_tar.py kuzu.tar.gz
 
       - name: Build wheels for Apple Silicon
         uses: pypa/cibuildwheel@v2.12.0

diff --git a/.github/workflows/windows-nodejs-workflow.yml b/.github/workflows/windows-nodejs-workflow.yml
@@ -2,35 +2,30 @@ name: Build-Windows-NodeJS-Module
 
 on:
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   build-windows-nodejs:
     runs-on: self-hosted-windows
     steps:
       - uses: actions/checkout@v3
 
-      - name: Create Node.js source distribution
+      - name: Install Node.js dependencies
         working-directory: tools/nodejs_api
         run: |
-          node clean
-          node package
-
-      - name: Extract tarball
-        working-directory: tools/nodejs_api
-        run: tar -xvzf kuzu-source.tar.gz
+          npm i
 
       - name: Build Node.js native module
-        working-directory: tools/nodejs_api/package
         shell: cmd
         run: |
           call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat"
-          npm i
+          make nodejs NUM_THREADS=48
 
       - name: Move Node.js native module
-        working-directory: tools/nodejs_api/package
+        working-directory: tools/nodejs_api/build
         run: ren kuzujs.node kuzujs-win32-x64.node
 
       - uses: actions/upload-artifact@v3
         with:
           name: windows-nodejs-module
-          path: tools/nodejs_api/package/kuzujs-win32-x64.node
+          path: tools/nodejs_api/build/kuzujs-win32-x64.node