From 81fef9bcc311a10e9629475fc5c5c6257972a497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=9B=A7=E5=9B=A7?= Date: Mon, 10 Jul 2023 11:49:05 -0400 Subject: [PATCH] Add CI pipeline for build and deploy everything (#1796) * X * X * Node.js publish attempt * Try to fix Windows Node.js workflow * Use backslash for Windows path * Remove path * Typo fix * Increase Windows Node.js build threads * Add missing working dir * Setup global env * Fix JAR path * Fix version number * Also upload Node.js deploy archive * Auto detect version for python API * Create source dist with version number * Test sdist * Add deploy * Remove version from bin workflows * Remove unused env * OK * Remove -sdist * Add precompiled bin workflow * Try to fix Windows CI precompile * Remove cleanup * Format yaml * Reorder * Minor fix --- .github/workflows/build-and-deploy.yml | 179 +++++++++++++++++- .github/workflows/linux-nodejs-workflow.yml | 1 + .../linux-precompiled-bin-workflow.yml | 16 +- .github/workflows/linux-wheel-workflow.yml | 9 +- .github/workflows/mac-nodejs-workflow.yml | 1 + .../mac-precompiled-bin-workflow.yml | 20 +- .github/workflows/mac-wheel-workflow.yml | 12 +- .github/workflows/windows-nodejs-workflow.yml | 17 +- .../windows-precompiled-bin-workflow.yml | 21 +- .github/workflows/windows-wheel-workflow.yml | 12 +- CMakeLists.txt | 2 +- scripts/pip-package/build_all_packages.sh | 2 +- scripts/pip-package/package_tar.py | 21 +- scripts/pip-package/setup.py | 2 +- 14 files changed, 221 insertions(+), 94 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index e02abcf481..b393fef433 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -27,33 +27,192 @@ jobs: steps: - uses: actions/download-artifact@v3 with: - name: libkuzu-java-win-x86_64 + name: libkuzu-java-osx-x86_64 path: java-bins - uses: actions/download-artifact@v3 with: - name: libkuzu-java-osx-x86_64 + name: libkuzu-java-osx-arm64 path: java-bins - uses: actions/download-artifact@v3 with: - name: libkuzu-java-osx-arm64 + name: kuzu-linux-jar path: java-bins - uses: actions/download-artifact@v3 with: - name: kuzu-linux-jar + name: libkuzu-java-win-x86_64 path: java-bins - name: Add Java libs to jar run: | - ls -alR java-bins - jar uf java-bins/kuzu-linux-jar/kuzu_java.jar java-bins/libkuzu-java-win-x86_64/libkuzu_java_native.so_windows_amd64 - jar uf java-bins/kuzu-linux-jar/kuzu_java.jar java-bins/libkuzu-java-osx-x86_64/libkuzu_java_native.so_osx_amd64 - jar uf java-bins/kuzu-linux-jar/kuzu_java.jar java-bins/libkuzu-java-osx-arm64/libkuzu_java_native.so_osx_arm64 - + jar uf kuzu_java.jar libkuzu_java_native.so_windows_amd64 + jar uf kuzu_java.jar libkuzu_java_native.so_osx_arm64 + jar uf kuzu_java.jar libkuzu_java_native.so_osx_amd64 + working-directory: java-bins + - name: Upload jar uses: actions/upload-artifact@v3 with: name: kuzu-deploy-jar - path: java-bins/kuzu-linux-jar/kuzu_java.jar + path: java-bins/kuzu_java.jar + + build-nodejs-mac: + uses: ./.github/workflows/mac-nodejs-workflow.yml + secrets: inherit + + build-nodejs-linux: + uses: ./.github/workflows/linux-nodejs-workflow.yml + secrets: inherit + + build-nodejs-windows: + uses: ./.github/workflows/windows-nodejs-workflow.yml + secrets: inherit + + deploy-nodejs: + needs: [build-nodejs-mac, build-nodejs-linux, build-nodejs-windows] + runs-on: ubuntu-latest + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_JS_TOKEN }} + steps: + - uses: actions/checkout@v3 + + - name: Create prebuilt folder + run: mkdir -p tools/nodejs_api/prebuilt + + - uses: actions/download-artifact@v3 + with: + name: mac-nodejs-module + path: tools/nodejs_api/prebuilt + + - uses: actions/download-artifact@v3 + with: + name: linux-nodejs-module + path: tools/nodejs_api/prebuilt + + - uses: actions/download-artifact@v3 + with: + name: windows-nodejs-module + path: tools/nodejs_api/prebuilt + + - uses: actions/setup-node@v3 + with: + node-version: "16" + registry-url: "https://registry.npmjs.org" + + - name: Package Node.js API with prebuilt binaries + run: node package + working-directory: tools/nodejs_api + + - name: Show tarball contents + run: tar -tvf kuzu-source.tar.gz + working-directory: tools/nodejs_api + + - name: Upload tarball + uses: actions/upload-artifact@v3 + with: + name: kuzu-deploy-nodejs + path: tools/nodejs_api/kuzu-source.tar.gz + + - name: Deploy to npm.js dry run + if: ${{ github.event.inputs.isDeploy != 'true' }} + run: npm publish kuzu-source.tar.gz --access public --dry-run + working-directory: tools/nodejs_api + + - name: Deploy to npm.js + if: ${{ github.event.inputs.isDeploy == 'true' }} + run: npm publish kuzu-source.tar.gz --access public + working-directory: tools/nodejs_api + + build-wheel-mac: + uses: ./.github/workflows/mac-wheel-workflow.yml + secrets: inherit + + build-wheel-linux: + uses: ./.github/workflows/linux-wheel-workflow.yml + secrets: inherit + + build-wheel-windows: + uses: ./.github/workflows/windows-wheel-workflow.yml + secrets: inherit + + package-python-sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Package Python sdist + run: python package_tar.py + working-directory: scripts/pip-package + + - name: Upload tarball + uses: actions/upload-artifact@v3 + with: + name: python-sdist + path: scripts/pip-package/*.tar.gz + + deploy-python: + needs: + [ + build-wheel-mac, + build-wheel-linux, + build-wheel-windows, + package-python-sdist, + ] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v3 + with: + name: macos-wheels + path: dist + + - uses: actions/download-artifact@v3 + with: + name: linux-wheels + path: dist + + - uses: actions/download-artifact@v3 + with: + name: windows-wheels + path: dist + + - uses: actions/download-artifact@v3 + with: + name: python-sdist + path: dist + + - name: List wheels + run: ls -l + working-directory: dist + + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: kuzu-deploy-wheels + path: dist/* + + - name: Deploy to PyPI test + if: ${{ github.event.inputs.isDeploy != 'true' }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TEST_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + + - name: Deploy to PyPI + if: ${{ github.event.inputs.isDeploy == 'true' }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_TOKEN }} + + build-precompiled-bin-mac: + uses: ./.github/workflows/mac-precompiled-bin-workflow.yml + secrets: inherit + + build-precompiled-bin-linux: + uses: ./.github/workflows/linux-precompiled-bin-workflow.yml + secrets: inherit + + build-precompiled-bin-windows: + uses: ./.github/workflows/windows-precompiled-bin-workflow.yml + secrets: inherit diff --git a/.github/workflows/linux-nodejs-workflow.yml b/.github/workflows/linux-nodejs-workflow.yml index 05cc10e39a..723f0775b2 100644 --- a/.github/workflows/linux-nodejs-workflow.yml +++ b/.github/workflows/linux-nodejs-workflow.yml @@ -2,6 +2,7 @@ name: Build-Linux-NodeJS-Module on: workflow_dispatch: + workflow_call: jobs: build-linux-nodejs: diff --git a/.github/workflows/linux-precompiled-bin-workflow.yml b/.github/workflows/linux-precompiled-bin-workflow.yml index af08048655..7ba37b926f 100644 --- a/.github/workflows/linux-precompiled-bin-workflow.yml +++ b/.github/workflows/linux-precompiled-bin-workflow.yml @@ -2,14 +2,7 @@ name: Build-Linux-Precompiled-Binaries on: workflow_dispatch: - inputs: - packageVersion: - description: "Version of output package" - required: true - default: "0.0.1" - -env: - PACKAGE_VERSION: ${{ github.event.inputs.packageVersion }} + workflow_call: jobs: build-precompiled-bin: @@ -26,7 +19,7 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: libkuzu-${{ github.event.inputs.packageVersion }}-linux-x86_64 + name: libkuzu-linux-x86_64 path: | ./scripts/pre-compiled-bins/kuzu.h ./scripts/pre-compiled-bins/kuzu.hpp @@ -34,8 +27,5 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: kuzu_cli-${{ github.event.inputs.packageVersion }}-linux-x86_64 + name: kuzu_cli-linux-x86_64 path: ./scripts/pre-compiled-bins/kuzu - - - name: Clean up - run: rm -rf ./scripts/pre-compiled-bins/kuzu ./scripts/pre-compiled-bins/headers ./scripts/pre-compiled-bins/libkuzu.so /scripts/pre-compiled-bins/kuzu.h /scripts/pre-compiled-bins/kuzu.hpp diff --git a/.github/workflows/linux-wheel-workflow.yml b/.github/workflows/linux-wheel-workflow.yml index 3512ce38f5..82c0b9f490 100644 --- a/.github/workflows/linux-wheel-workflow.yml +++ b/.github/workflows/linux-wheel-workflow.yml @@ -2,14 +2,7 @@ name: Build-Linux-Wheels on: workflow_dispatch: - inputs: - pythonPackageVersion: - description: "Version of output Python package" - required: true - default: "0.0.1" - -env: - PYTHON_PACKAGE_VERSION: ${{ github.event.inputs.pythonPackageVersion }} + workflow_call: jobs: build-linux-wheels: diff --git a/.github/workflows/mac-nodejs-workflow.yml b/.github/workflows/mac-nodejs-workflow.yml index 5edc11ce1f..67e6f87956 100644 --- a/.github/workflows/mac-nodejs-workflow.yml +++ b/.github/workflows/mac-nodejs-workflow.yml @@ -2,6 +2,7 @@ name: Build-Mac-NodeJS-Module on: workflow_dispatch: + workflow_call: jobs: build-mac-nodejs: diff --git a/.github/workflows/mac-precompiled-bin-workflow.yml b/.github/workflows/mac-precompiled-bin-workflow.yml index 4a4db15e79..43acf21cdc 100644 --- a/.github/workflows/mac-precompiled-bin-workflow.yml +++ b/.github/workflows/mac-precompiled-bin-workflow.yml @@ -2,14 +2,7 @@ name: Build-Mac-Precompiled-Binaries on: workflow_dispatch: - inputs: - packageVersion: - description: "Version of output package" - required: true - default: "0.0.1" - -env: - PACKAGE_VERSION: ${{ github.event.inputs.packageVersion }} + workflow_call: jobs: build-precompiled-bin: @@ -29,7 +22,7 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: libkuzu-${{ github.event.inputs.packageVersion }}-osx-arm64 + name: libkuzu-osx-arm64 path: | ./scripts/pre-compiled-bins/kuzu.h ./scripts/pre-compiled-bins/kuzu.hpp @@ -37,7 +30,7 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: kuzu_cli-${{ github.event.inputs.packageVersion }}-osx-arm64 + name: kuzu_cli-osx-arm64 path: ./scripts/pre-compiled-bins/kuzu - name: Build precompiled binaries for Intel @@ -49,7 +42,7 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: libkuzu-${{ github.event.inputs.packageVersion }}-osx-x86_64 + name: libkuzu-osx-x86_64 path: | ./scripts/pre-compiled-bins/kuzu.h ./scripts/pre-compiled-bins/kuzu.hpp @@ -57,8 +50,5 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: kuzu_cli-${{ github.event.inputs.packageVersion }}-osx-x86_64 + name: kuzu_cli-osx-x86_64 path: ./scripts/pre-compiled-bins/kuzu - - - name: Clean up - run: rm -rf ./scripts/pre-compiled-bins/kuzu ./scripts/pre-compiled-bins/headers ./scripts/pre-compiled-bins/libkuzu.dylib ./scripts/pre-compiled-bins/kuzu.h ./scripts/pre-compiled-bins/kuzu.hpp diff --git a/.github/workflows/mac-wheel-workflow.yml b/.github/workflows/mac-wheel-workflow.yml index 70b5168e41..5c8572a9c3 100644 --- a/.github/workflows/mac-wheel-workflow.yml +++ b/.github/workflows/mac-wheel-workflow.yml @@ -2,26 +2,20 @@ name: Build-Mac-Wheels on: workflow_dispatch: - inputs: - pythonPackageVersion: - description: "Version of output Python package" - required: true - default: "0.0.1" - -env: - PYTHON_PACKAGE_VERSION: ${{ github.event.inputs.pythonPackageVersion }} + workflow_call: jobs: build-wheels: runs-on: self-hosted-mac-arm steps: - uses: actions/checkout@v3 + - name: Create source distribution working-directory: ./scripts/pip-package/ run: | rm -rf wheelhouse kuzu.tar.gz mkdir wheelhouse - python3 package_tar.py + python3 package_tar.py kuzu.tar.gz - name: Build wheels for Apple Silicon uses: pypa/cibuildwheel@v2.12.0 diff --git a/.github/workflows/windows-nodejs-workflow.yml b/.github/workflows/windows-nodejs-workflow.yml index 334272ec94..d2ed6d26fc 100644 --- a/.github/workflows/windows-nodejs-workflow.yml +++ b/.github/workflows/windows-nodejs-workflow.yml @@ -2,6 +2,7 @@ name: Build-Windows-NodeJS-Module on: workflow_dispatch: + workflow_call: jobs: build-windows-nodejs: @@ -9,28 +10,22 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Create Node.js source distribution + - name: Install Node.js dependencies working-directory: tools/nodejs_api run: | - node clean - node package - - - name: Extract tarball - working-directory: tools/nodejs_api - run: tar -xvzf kuzu-source.tar.gz + npm i - name: Build Node.js native module - working-directory: tools/nodejs_api/package shell: cmd run: | call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat" - npm i + make nodejs NUM_THREADS=48 - name: Move Node.js native module - working-directory: tools/nodejs_api/package + working-directory: tools/nodejs_api/build run: ren kuzujs.node kuzujs-win32-x64.node - uses: actions/upload-artifact@v3 with: name: windows-nodejs-module - path: tools/nodejs_api/package/kuzujs-win32-x64.node + path: tools/nodejs_api/build/kuzujs-win32-x64.node diff --git a/.github/workflows/windows-precompiled-bin-workflow.yml b/.github/workflows/windows-precompiled-bin-workflow.yml index 92eb7d8668..e022e616d9 100644 --- a/.github/workflows/windows-precompiled-bin-workflow.yml +++ b/.github/workflows/windows-precompiled-bin-workflow.yml @@ -2,14 +2,7 @@ name: Build-Windows-Precompiled-Binaries on: workflow_dispatch: - inputs: - packageVersion: - description: "Version of output package" - required: true - default: "0.0.1" - -env: - PACKAGE_VERSION: ${{ github.event.inputs.packageVersion }} + workflow_call: jobs: build-precompiled-bin: @@ -21,12 +14,15 @@ jobs: run: python.exe -m pip install networkx --user - name: Build precompiled binaries - run: python.exe build.py + shell: cmd + run: | + call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvars64.bat" + python.exe build.py working-directory: ./scripts/pre-compiled-bins/ - uses: actions/upload-artifact@v3 with: - name: libkuzu-${{ github.event.inputs.packageVersion }}-windows-x86_64 + name: libkuzu-windows-x86_64 path: | ./scripts/pre-compiled-bins/kuzu.h ./scripts/pre-compiled-bins/kuzu.hpp @@ -35,8 +31,5 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: kuzu_cli-${{ github.event.inputs.packageVersion }}-windows-x86_64 + name: kuzu_cli-windows-x86_64 path: ./scripts/pre-compiled-bins/kuzu.exe - - - name: Clean up - run: rm -rf ./scripts/pre-compiled-bins/kuzu ./scripts/pre-compiled-bins/headers ./scripts/pre-compiled-bins/kuzu_shared.dll ./scripts/pre-compiled-bins/kuzu_shared.lib ./scripts/pre-compiled-bins/kuzu.h ./scripts/pre-compiled-bins/kuzu.hpp diff --git a/.github/workflows/windows-wheel-workflow.yml b/.github/workflows/windows-wheel-workflow.yml index c2e4712c08..cde26572d5 100644 --- a/.github/workflows/windows-wheel-workflow.yml +++ b/.github/workflows/windows-wheel-workflow.yml @@ -2,14 +2,7 @@ name: Build-Windows-Wheels on: workflow_dispatch: - inputs: - pythonPackageVersion: - description: "Version of output Python package" - required: true - default: "0.0.1" - -env: - PYTHON_PACKAGE_VERSION: ${{ github.event.inputs.pythonPackageVersion }} + workflow_call: jobs: build-wheels: @@ -23,7 +16,7 @@ jobs: run: | if exist wheelhouse rmdir /s /q wheelhouse mkdir wheelhouse - python.exe package_tar.py + python.exe package_tar.py kuzu.tar.gz # Manually set up cibuildwheel instead of importing their # action since there seems to be an issue with quoting @@ -51,4 +44,5 @@ jobs: - uses: actions/upload-artifact@v3 with: + name: windows-wheels path: ./scripts/pip-package/wheelhouse/*.whl diff --git a/CMakeLists.txt b/CMakeLists.txt index 90fd5d95f6..6f2b7d8284 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.11) -project(Kuzu VERSION 0.0.5.1 LANGUAGES CXX) +project(Kuzu VERSION 0.0.5 LANGUAGES CXX) find_package(Threads REQUIRED) diff --git a/scripts/pip-package/build_all_packages.sh b/scripts/pip-package/build_all_packages.sh index e3886ca486..33fe99cf34 100644 --- a/scripts/pip-package/build_all_packages.sh +++ b/scripts/pip-package/build_all_packages.sh @@ -3,7 +3,7 @@ PLATFORM="manylinux2014_x86_64" chmod +x ./package_tar.sh -rm -rf wheelhouse kuzu.tar.gz && /opt/python/cp311-cp311/bin/python ./package_tar.py +rm -rf wheelhouse kuzu.tar.gz && /opt/python/cp311-cp311/bin/python ./package_tar.py kuzu.tar.gz mkdir wheelhouse # Build wheels, excluding pypy platforms and python 3.6 diff --git a/scripts/pip-package/package_tar.py b/scripts/pip-package/package_tar.py index 72d40c906f..511c145f20 100755 --- a/scripts/pip-package/package_tar.py +++ b/scripts/pip-package/package_tar.py @@ -7,7 +7,22 @@ import tarfile from tempfile import TemporaryDirectory +base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +def _get_kuzu_version(): + cmake_file = os.path.abspath(os.path.join(base_dir, '..', 'CMakeLists.txt')) + with open(cmake_file) as f: + for line in f: + if line.startswith('project(Kuzu VERSION'): + return line.split(' ')[2].strip() + if __name__ == "__main__": + if len(sys.argv) == 2: + file_name = sys.argv[1] + else: + file_name = "kuzu-%s.tar.gz" % _get_kuzu_version() + print("Creating %s..." % file_name) + with TemporaryDirectory() as tempdir: subprocess.check_call( [ @@ -24,6 +39,8 @@ with tarfile.open(os.path.join(tempdir, "kuzu-source.tar")) as tar: tar.extractall(path=os.path.join(tempdir, "kuzu-source")) + + os.remove(os.path.join(tempdir, "kuzu-source.tar")) os.makedirs(os.path.join(tempdir, "kuzu")) for path in ["setup.py", "setup.cfg", "MANIFEST.in"]: @@ -34,5 +51,5 @@ subprocess.check_call([sys.executable, "setup.py", "egg_info"], cwd=tempdir) - with tarfile.open("kuzu.tar.gz", "w:gz") as sdist: - sdist.add(tempdir, "kuzu") + with tarfile.open(file_name, "w:gz") as sdist: + sdist.add(tempdir, "sdist") diff --git a/scripts/pip-package/setup.py b/scripts/pip-package/setup.py index 1a85303de1..d4b8bff3e7 100644 --- a/scripts/pip-package/setup.py +++ b/scripts/pip-package/setup.py @@ -23,7 +23,7 @@ def _get_kuzu_version(): return line.split(' ')[2].strip() -kuzu_version = os.environ['PYTHON_PACKAGE_VERSION'] if 'PYTHON_PACKAGE_VERSION' in os.environ else _get_kuzu_version() +kuzu_version = _get_kuzu_version() print("The version of this build is %s" % kuzu_version)