-
Notifications
You must be signed in to change notification settings - Fork 90
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1266 from kuzudb/precompile-bin-pipeline
Fix pipeline for building C++ lib and CLI
- Loading branch information
Showing
9 changed files
with
180 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,21 +1,21 @@ | ||
| FROM quay.io/pypa/manylinux2014_x86_64 | ||
|
|
||
| ENV RUNNER_ALLOW_RUNASROOT=1 | ||
|
|
||
|
|
||
| # Install dependencies | ||
| RUN yum --disablerepo=epel -y upgrade curl ca-certificates | ||
| RUN yum update -y | ||
| RUN yum install -y cmake nodejs jq git perl-Digest-SHA libicu libicu-devel | ||
|
|
||
| RUN useradd --create-home runner | ||
| USER runner | ||
|
|
||
| # Install GitHub action runner | ||
| RUN mkdir /actions-runner | ||
| WORKDIR /actions-runner | ||
| RUN curl -o actions-runner-linux-x64-2.298.2.tar.gz -L https://github.com/actions/runner/releases/download/v2.298.2/actions-runner-linux-x64-2.298.2.tar.gz | ||
| RUN echo "0bfd792196ce0ec6f1c65d2a9ad00215b2926ef2c416b8d97615265194477117 actions-runner-linux-x64-2.298.2.tar.gz" | shasum -a 256 | ||
| RUN tar xzf ./actions-runner-linux-x64-2.298.2.tar.gz | ||
| RUN mkdir /home/runner/actions-runner | ||
| WORKDIR /home/runner/actions-runner | ||
| RUN curl -o actions-runner-linux-x64-2.301.1.tar.gz -L https://github.com/actions/runner/releases/download/v2.301.1/actions-runner-linux-x64-2.301.1.tar.gz | ||
| RUN echo "3ee9c3b83de642f919912e0594ee2601835518827da785d034c1163f8efdf907 actions-runner-linux-x64-2.301.1.tar.gz" | shasum -a 256 -c | ||
| RUN tar xzf ./actions-runner-linux-x64-2.301.1.tar.gz | ||
|
|
||
| COPY listener.sh listener.sh | ||
| COPY --chown=runner:runner listener.sh listener.sh | ||
| RUN chmod +x listener.sh | ||
|
|
||
| ENTRYPOINT ["./listener.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| # Intro | ||
| The script in this directory builds pre-compiled library and CLI and collects header for release. It generates the following files: | ||
| - `headers`: the headers for release in a directory; | ||
| - `kuzu.h`: the merged header for release; | ||
| - `libkuzu.so` / `libkuzu.dylib`: the shared library for C++ API; | ||
| - `kuzu`: the database shell. | ||
| # Usage: | ||
| - Install dependency: `pip3 install networkx`. | ||
| - Run build: `python3 build.py`. | ||
| # Run on CI: | ||
| Currently we use the same environment to build the pre-compiled binaries as Python wheels. Please refer to the instructions in [this file](../pip-package/README.md). |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| import subprocess | ||
| import os | ||
| import sys | ||
| import logging | ||
| import shutil | ||
| import multiprocessing | ||
|
|
||
| concurrency = multiprocessing.cpu_count() | ||
|
|
||
| logging.basicConfig(level=logging.DEBUG) | ||
|
|
||
| base_dir = os.path.dirname(__file__) | ||
| workspace_root = os.path.realpath(os.path.join(base_dir, "..", "..")) | ||
| env_vars = os.environ.copy() | ||
| env_vars['PYTHON_BIN_PATH'] = sys.executable | ||
| if sys.platform == "linux": | ||
| env_vars["CC"] = "gcc" | ||
| env_vars["CXX"] = "g++" | ||
| elif sys.platform == "darwin": | ||
| env_vars["CC"] = "clang" | ||
| env_vars["CXX"] = "clang++" | ||
|
|
||
|
|
||
| def cleanup(): | ||
| logging.info("Cleaning up workspace...") | ||
| try: | ||
| subprocess.run(['make', 'clean-all'], cwd=workspace_root, | ||
| check=True, env=env_vars) | ||
| shutil.rmtree(os.path.join(base_dir, "headers"), ignore_errors=True) | ||
| for f in ["kuzu", "kuzu.h", "libkuzu.so", "libkuzu.dylib"]: | ||
| try: | ||
| os.remove(os.path.join(base_dir, f)) | ||
| except OSError: | ||
| pass | ||
| except subprocess.CalledProcessError as e: | ||
| return_code = e.returncode | ||
| logging.error("Failed with return code %d" % return_code) | ||
| sys.exit(1) | ||
| logging.info("Workspace cleaned up") | ||
|
|
||
|
|
||
| def build(): | ||
| logging.info("Building kuzu...") | ||
| if sys.platform == 'darwin': | ||
| archflags = os.getenv("ARCHFLAGS", "") | ||
|
|
||
| if len(archflags) > 0: | ||
| logging.info("The ARCHFLAGS is set to '%s'." % | ||
| archflags) | ||
| if archflags == "-arch arm64": | ||
| env_vars['CMAKE_OSX_ARCHITECTURES'] = "arm64" | ||
| elif archflags == "-arch x86_64": | ||
| env_vars['CMAKE_OSX_ARCHITECTURES'] = "x86_64" | ||
| else: | ||
| logging.info( | ||
| "The ARCHFLAGS is not valid and will be ignored.") | ||
| else: | ||
| logging.info("The ARCHFLAGS is not set.") | ||
|
|
||
| deploy_target = os.getenv("MACOSX_DEPLOYMENT_TARGET", "") | ||
| if len(deploy_target) > 0: | ||
| logging.info("The deployment target is set to '%s'." % | ||
| deploy_target) | ||
| env_vars['CMAKE_OSX_DEPLOYMENT_TARGET'] = deploy_target | ||
|
|
||
| full_cmd = ['make', 'release', 'NUM_THREADS=%d' % concurrency] | ||
| logging.info("Running command: %s" % full_cmd) | ||
| try: | ||
| subprocess.run(full_cmd, cwd=workspace_root, | ||
| check=True, env=env_vars) | ||
| except subprocess.CalledProcessError as e: | ||
| return_code = e.returncode | ||
| logging.error("Failed with return code %d" % return_code) | ||
| sys.exit(1) | ||
| logging.info("Build completed") | ||
|
|
||
|
|
||
| def collect_and_merge_headers(): | ||
| logging.info("Running script to collect and merge headers...") | ||
| try: | ||
| subprocess.run([sys.executable, 'collect_files.py'], | ||
| cwd=base_dir) | ||
| subprocess.run([sys.executable, 'merge_headers.py'], | ||
| cwd=base_dir) | ||
| except subprocess.CalledProcessError as e: | ||
| return_code = e.returncode | ||
| logging.error("Failed with return code %d" % return_code) | ||
| sys.exit(1) | ||
| logging.info("Files collected") | ||
|
|
||
|
|
||
| def collect_binaries(): | ||
| logging.info("Collecting binaries...") | ||
| so_path = os.path.join(workspace_root, "build", | ||
| "release", "src", "libkuzu.so") | ||
| dylib_path = os.path.join(workspace_root, "build", | ||
| "release", "src", "libkuzu.dylib") | ||
| so_exists = os.path.exists(so_path) | ||
| dylib_exists = os.path.exists(dylib_path) | ||
| if not so_exists and not dylib_exists: | ||
| logging.error("No shared object file found") | ||
| sys.exit(1) | ||
| if so_exists: | ||
| shutil.copy(so_path, os.path.join(base_dir, "libkuzu.so")) | ||
| logging.info("Copied libkuzu.so") | ||
| if dylib_exists: | ||
| shutil.copy(dylib_path, os.path.join(base_dir, "libkuzu.dylib")) | ||
| logging.info("Copied libkuzu.so") | ||
| shell_path = os.path.join(workspace_root, "build", | ||
| "release", "tools", "shell", "kuzu_shell") | ||
| if not os.path.exists(shell_path): | ||
| logging.error("No shell binary found") | ||
| sys.exit(1) | ||
| shutil.copy(shell_path, os.path.join(base_dir, "kuzu")) | ||
| logging.info("Copied kuzu") | ||
| logging.info("Binaries collected") | ||
|
|
||
|
|
||
| def main(): | ||
| cleanup() | ||
| build() | ||
| collect_binaries() | ||
| collect_and_merge_headers() | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters