Split CI build pipeline for X86 and ARM Mac (#1926)

* Split CI build pipeline for X86 and ARM Mac * Fix JAVA_HOME for macOS X64 * Temp override project version for testing * Revert version number
kuzudb · Aug 12, 2023 · 1f0d8ac · 1f0d8ac
1 parent e48ea4f
commit 1f0d8ac
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 15 deletions.
diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml
@@ -83,7 +83,12 @@ jobs:
 
       - uses: actions/download-artifact@v3
         with:
-          name: mac-nodejs-module
+          name: mac-nodejs-module-arm64
+          path: tools/nodejs_api/prebuilt
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: mac-nodejs-module-x86_64
           path: tools/nodejs_api/prebuilt
 
       - uses: actions/download-artifact@v3
@@ -164,7 +169,12 @@ jobs:
     steps:
       - uses: actions/download-artifact@v3
         with:
-          name: macos-wheels
+          name: macos-wheels-arm64
+          path: dist
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: macos-wheels-x86_64
           path: dist
 
       - uses: actions/download-artifact@v3

diff --git a/.github/workflows/mac-java-workflow.yml b/.github/workflows/mac-java-workflow.yml
@@ -8,31 +8,31 @@ jobs:
     runs-on: self-hosted-mac-arm
     steps:
       - uses: actions/checkout@v3
-    
+
       - name: Build Java lib for Apple Silicon
         run: |
           arch -arm64 env JAVA_HOME=$(/usr/libexec/java_home) make java NUM_THREADS=8
         env:
           MACOSX_DEPLOYMENT_TARGET: 11.0
           ARCHFLAGS: "-arch arm64"
-          
+
       - uses: actions/upload-artifact@v3
         with:
           name: libkuzu-java-osx-arm64
           path: tools/java_api/build/libkuzu_java_native*
-    
+
   build-mac-java-x86:
-    runs-on: macos-latest
+    runs-on: self-hosted-mac-x64
     steps:
       - uses: actions/checkout@v3
-    
+
       - name: Build Java lib for Intel
         run: |
-          env JAVA_HOME=$JAVA_HOME_11_X64 make java NUM_THREADS=3
+          env JAVA_HOME=$(/usr/libexec/java_home) make java NUM_THREADS=48
         env:
           MACOSX_DEPLOYMENT_TARGET: 10.15
           ARCHFLAGS: "-arch x86_64"
-      
+
       - uses: actions/upload-artifact@v3
         with:
           name: libkuzu-java-osx-x86_64

diff --git a/.github/workflows/mac-nodejs-workflow.yml b/.github/workflows/mac-nodejs-workflow.yml
@@ -5,7 +5,7 @@ on:
   workflow_call:
 
 jobs:
-  build-mac-nodejs:
+  build-mac-nodejs-arm64:
     runs-on: self-hosted-mac-arm
     steps:
       - uses: actions/checkout@v3
@@ -31,10 +31,26 @@ jobs:
         working-directory: tools/nodejs_api/
         run: mv package/kuzujs.node kuzujs-darwin-arm64.node
 
+      - uses: actions/upload-artifact@v3
+        with:
+          name: mac-nodejs-module-arm64
+          path: tools/nodejs_api/kuzujs-darwin-*.node
+
       - name: Clean up
         working-directory: tools/nodejs_api/
         run: rm -rf package
 
+  build-mac-nodejs-x86_64:
+    runs-on: self-hosted-mac-x64
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create Node.js source distribution
+        working-directory: tools/nodejs_api
+        run: |
+          rm -rf kuzu-source.tar.gz package *.node
+          node package
+
       - name: Extract tarball
         working-directory: tools/nodejs_api
         run: tar -xvzf kuzu-source.tar.gz
@@ -52,5 +68,9 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: mac-nodejs-module
+          name: mac-nodejs-module-x86_64
           path: tools/nodejs_api/kuzujs-darwin-*.node
+
+      - name: Clean up
+        working-directory: tools/nodejs_api/
+        run: rm -rf package
diff --git a/.github/workflows/mac-precompiled-bin-workflow.yml b/.github/workflows/mac-precompiled-bin-workflow.yml
@@ -5,7 +5,7 @@ on:
   workflow_call:
 
 jobs:
-  build-precompiled-bin:
+  build-precompiled-bin-arm64:
     runs-on: self-hosted-mac-arm
     steps:
       - uses: actions/checkout@v3
@@ -33,6 +33,14 @@ jobs:
           name: kuzu_cli-osx-arm64
           path: ./scripts/pre-compiled-bins/kuzu
 
+  build-precompiled-bin-x86_64:
+    runs-on: self-hosted-mac-x64
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install networkx
+        run: python3 -m pip install networkx --user
+
       - name: Build precompiled binaries for Intel
         run: python3 build.py
         working-directory: ./scripts/pre-compiled-bins/

diff --git a/.github/workflows/mac-wheel-workflow.yml b/.github/workflows/mac-wheel-workflow.yml
@@ -5,11 +5,11 @@ on:
   workflow_call:
 
 jobs:
-  build-wheels:
+  build-wheels-arm64:
     runs-on: self-hosted-mac-arm
     steps:
       - uses: actions/checkout@v3
-      
+
       - name: Create source distribution
         working-directory: ./scripts/pip-package/
         run: |
@@ -28,6 +28,23 @@ jobs:
           package-dir: ./scripts/pip-package/kuzu.tar.gz
           output-dir: ./scripts/pip-package/wheelhouse
 
+      - uses: actions/upload-artifact@v3
+        with:
+          name: macos-wheels-arm64
+          path: ./scripts/pip-package/wheelhouse/*.whl
+
+  build-wheels-x86_64:
+    runs-on: self-hosted-mac-x64
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Create source distribution
+        working-directory: ./scripts/pip-package/
+        run: |
+          rm -rf wheelhouse kuzu.tar.gz
+          mkdir wheelhouse
+          python3 package_tar.py kuzu.tar.gz
+
       - name: Build wheels for Intel
         uses: pypa/cibuildwheel@v2.12.0
         env:
@@ -41,5 +58,5 @@ jobs:
 
       - uses: actions/upload-artifact@v3
         with:
-          name: macos-wheels
+          name: macos-wheels-x86_64
           path: ./scripts/pip-package/wheelhouse/*.whl