diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml index 8b0263b282e2..6d860f2d0830 100644 --- a/.github/workflows/build-graphscope-wheels-macos.yml +++ b/.github/workflows/build-graphscope-wheels-macos.yml @@ -19,10 +19,10 @@ concurrency: jobs: build-wheels: if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') - runs-on: macos-12 + runs-on: macos-13 strategy: matrix: - python-version: ['3.11'] + python-version: ['3.9'] steps: - uses: actions/checkout@v4 @@ -30,76 +30,30 @@ jobs: submodules: true - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install Dependencies - run: | - brew update || true - brew install bash coreutils - - alias bash=$(brew --prefix)/bin/bash - - export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH - - sudo mkdir /opt/graphscope - sudo chown -R $(id -u):$(id -g) /opt/graphscope - - # Not sure when the first installation would fail, saying that the lock of python3.10 is taken - # However the second trial would success. - python3 -m pip install click - python3 gsctl.py install-deps dev || true - python3 gsctl.py install-deps dev - echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env - - name: Install JDK 11 uses: actions/setup-java@v4 with: distribution: 'zulu' # See 'Supported distributions' for available options java-version: '11' - - name: Reinstall glog + - name: Install Dependencies run: | - # Essential cause glog installed by brew is broken and produce corrupted messages - brew uninstall --ignore-dependencies glog - . ~/.graphscope_env - git clone https://github.com/google/glog.git -b v0.6.0 - cd glog - - cmake . -DWITH_TLS=OFF -DWITH_SYMBOLIZE=OFF -DWITH_UNWIND=OFF -DWITH_GTEST=OFF -DBUILD_TESTING=OFF - make -j - sudo make install - - # make delocate-wheel happy - sudo mkdir -p /usr/local/opt/glog/lib - sudo ln -s /usr/local/lib/libglog*.dylib /usr/local/opt/glog/lib/ + python3 -c 'import sys; print(sys.version_info[:])' + export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH - - name: Install latest libgrape-lite - if: false - run: | - . ~/.graphscope_env - git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite - cd /tmp/libgrape-lite - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local - make -j4 - sudo make install + brew update || true + brew install bash coreutils + alias bash=$(brew --prefix)/bin/bash - - name: Install latest vineyard - if: false - run: | - source ~/.graphscope_env - python3 -m pip install libclang - git clone --single-branch -b v0.13.4 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d - cd /tmp/v6d - git submodule update --init - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DCMAKE_PREFIX_PATH=/usr/local \ - -DUSE_EXTERNAL_ETCD_LIBS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_VINEYARD_TESTS=OFF - make -j4 - sudo make install + python3 -m pip install click + # install the specific vineyard version: + # python3 gsctl.py install-deps dev --v6d-version + python3 gsctl.py install-deps dev + echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env - name: Build Server Wheel run: | @@ -115,8 +69,7 @@ jobs: # e.g. 0.15.0 -> 0.15.0a20220808 time=$(date "+%Y%m%d") version=$(cat ${GITHUB_WORKSPACE}/VERSION) - if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; - then + if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION; fi @@ -143,10 +96,10 @@ jobs: build-client-wheels: if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') - runs-on: macos-12 + runs-on: macos-13 strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 @@ -154,47 +107,22 @@ jobs: submodules: true - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install Dependencies run: | + python3 -c 'import sys; print(sys.version_info[:])' + export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH + brew update || true brew install bash coreutils alias bash=$(brew --prefix)/bin/bash - sudo mkdir /opt/graphscope - sudo chown -R $(id -u):$(id -g) /opt/graphscope python3 -m pip install click - python3 gsctl.py install-deps dev || true python3 gsctl.py install-deps dev - - - name: Install latest libgrape-lite - if: false - run: | - . ~/.graphscope_env - git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite - cd /tmp/libgrape-lite - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local - make -j4 - sudo make install - - - name: Install latest vineyard - if: false - run: | - . ~/.graphscope_env - python3 -m pip install libclang - git clone --single-branch -b v0.23.0 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d - cd /tmp/v6d - git submodule update --init - cmake . -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DCMAKE_PREFIX_PATH=/usr/local \ - -DUSE_EXTERNAL_ETCD_LIBS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_VINEYARD_TESTS=OFF - make -j4 - sudo make install + echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env - name: Build Client Wheels run: | @@ -234,7 +162,7 @@ jobs: needs: [build-wheels, build-client-wheels] strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/download-artifact@v4 @@ -252,7 +180,7 @@ jobs: tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz mv ${GITHUB_WORKSPACE}/artifacts/python/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - if [ "$PYTHON" == "3.8" ]; then + if [ "$PYTHON" == "3.9" ]; then tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz mv ${GITHUB_WORKSPACE}/artifacts/coordinator/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ fi @@ -277,12 +205,12 @@ jobs: python-test: if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }} - runs-on: macos-12 + runs-on: macos-13 needs: [build-wheels, build-client-wheels] strategy: fail-fast: false matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/download-artifact@v4 diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml index da6524b7d353..5bf0558997ff 100644 --- a/.github/workflows/gss.yml +++ b/.github/workflows/gss.yml @@ -94,6 +94,7 @@ jobs: - name: Groot with GOpt Integration Test run: | . ${HOME}/.graphscope_env + rm -rf /tmp/gstest || true git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git /tmp/gstest mv /tmp/gstest/flex/ldbc-sf01-long-date /tmp/gstest/ldbc cp -r flex/interactive/examples/movies /tmp/gstest/ diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index e04cf97b6dac..1de02edf06bb 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -574,8 +574,14 @@ install_patchelf() { # libgrape-lite install_libgrape_lite() { - if [[ -f "${install_prefix}/include/grape/grape.h" ]]; then - return 0 + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + if [[ -f "${HOMEBREW_PREFIX}/include/grape/grape.h" ]]; then + return 0 + fi + else + if [[ -f "${install_prefix}/include/grape/grape.h" ]]; then + return 0 + fi fi local branch=$1 pushd "${tempdir}" || exit @@ -599,7 +605,13 @@ install_vineyard() { python3 -m pip --no-cache-dir install pip -U --user python3 -m pip --no-cache-dir install libclang wheel auditwheel --user auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')") - sed -i 's/p.error/logger.warning/g' ${auditwheel_path} + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + BUILD_VINEYARD_GRAPH_WITH_GAR="OFF" + sed -i '' 's/p.error/logger.warning/g' ${auditwheel_path} + else + BUILD_VINEYARD_GRAPH_WITH_GAR="ON" + sed -i 's/p.error/logger.warning/g' ${auditwheel_path} + fi if [[ "${v6d_version}" != "v"* ]]; then directory="v6d" file="${directory}-${v6d_version}.tar.gz" @@ -619,10 +631,14 @@ install_vineyard() { -DBUILD_VINEYARD_TESTS=OFF \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \ - -DBUILD_VINEYARD_GRAPH_WITH_GAR=ON + -DBUILD_VINEYARD_GRAPH_WITH_GAR=${BUILD_VINEYARD_GRAPH_WITH_GAR} make -j$(nproc) make install - strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard* + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + strip "${V6D_PREFIX}"/bin/vineyard* + else + strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard* + fi pip3 install --no-cache -i https://pypi.org/simple -U "vineyard" "vineyard-io" cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/ set +e @@ -807,11 +823,12 @@ install_analytical_dependencies() { install_java_and_maven # install vineyard if [[ "${no_v6d}" != true ]]; then - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - brew install vineyard - else - install_vineyard - fi + install_vineyard + # if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + # brew install vineyard + # else + # install_vineyard + # fi fi } @@ -819,12 +836,17 @@ install_analytical_java_dependencies() { # llvm if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then brew install llvm || true # prevent the `brew link` failure + export CC=${HOMEBREW_PREFIX}/opt/llvm/bin/clang + export CXX=${HOMEBREW_PREFIX}/opt/llvm/bin/clang++ + export CPPFLAGS="${CPPFLAGS} -I${HOMEBREW_PREFIX}/opt/llvm/include" + export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC} elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then ${SUDO} apt-get install -y llvm-11-dev lld-11 clang-11 else if [[ "${OS_VERSION}" -eq "7" ]]; then ${SUDO} yum install -y llvm-toolset-7.0-clang-devel source /opt/rh/llvm-toolset-7.0/enable + export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/ else ${SUDO} yum install -y llvm-devel clang-devel lld fi @@ -865,7 +887,9 @@ install_interactive_dependencies() { rustc --version fi # opentelemetry - install_opentelemetry + if [[ "${OS_PLATFORM}" != *"Darwin"* ]]; then + install_opentelemetry + fi } install_learning_dependencies() { @@ -884,9 +908,9 @@ write_env_config() { } >> "${OUTPUT_ENV_FILE}" { if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl" - echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib" - echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib" + echo "export OPENSSL_ROOT_DIR=${HOMEBREW_PREFIX}/opt/openssl" + echo "export OPENSSL_LIBRARIES=${HOMEBREW_PREFIX}/opt/openssl/lib" + echo "export OPENSSL_SSL_LIBRARY=${HOMEBREW_PREFIX}/opt/openssl/lib/libssl.dylib" elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then if [[ "${OS_VERSION}" -eq "7" ]]; then echo "source /opt/rh/devtoolset-8/enable" @@ -916,11 +940,11 @@ write_env_config() { { if [[ "${for_analytical_java}" == true ]]; then if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - echo "export CC=${homebrew_prefix}/opt/llvm/bin/clang" - echo "export CXX=${homebrew_prefix}/opt/llvm/bin/clang++" + echo "export CC=${HOMEBREW_PREFIX}/opt/llvm/bin/clang" + echo "export CXX=${HOMEBREW_PREFIX}/opt/llvm/bin/clang++" echo "export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}" - echo "export LDFLAGS=\"-L${homebrew_prefix}/opt/llvm/lib\"" - echo "export CPPFLAGS=\"-I${homebrew_prefix}/opt/llvm/include\"" + echo "export LDFLAGS=\"-L${HOMEBREW_PREFIX}/opt/llvm/lib\"" + echo "export CPPFLAGS=\"-I${HOMEBREW_PREFIX}/opt/llvm/include\"" elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then echo "source /opt/rh/llvm-toolset-7.0/enable || true" echo "export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/"