From 486f4248947628ce648cb1b7fb17ba78ac4f2071 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Thu, 5 Sep 2024 11:57:52 +0800 Subject: [PATCH 1/8] Upgrade boost version to v1.75.0 for leaf --- python/graphscope/gsctl/scripts/install_deps.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index 9213e3d59bc3..33d0828fd4b0 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -196,7 +196,9 @@ function set_to_cn_url() { function fetch_source() { local url=$1 local file=$2 - curl -fsSL -o "${file}" "${url}/${file}" + command="curl -fsSL -o \"${file}\" \"${url}/${file}\"" + info ${command} + eval ${command} } function download_and_untar() { @@ -285,15 +287,15 @@ install_glog() { rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" } -# boost for centos +# boost with leaf for centos and ubuntu install_boost() { if [[ -f "${install_prefix}/include/boost/version.hpp" ]]; then return 0 fi pushd "${tempdir}" || exit - directory="boost_1_74_0" + directory="boost_1_75_0" file="${directory}.tar.gz" - url="https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source" + url="https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source" url=$(set_to_cn_url ${url}) download_and_untar "${url}" "${file}" "${directory}" pushd ${directory} || exit @@ -555,8 +557,6 @@ install_vineyard() { fi pushd ${directory} || exit - # make sure it complain loudly if installing vineyard fails - set -e cmake . -DCMAKE_PREFIX_PATH="${install_prefix}" \ -DCMAKE_INSTALL_PREFIX="${V6D_PREFIX}" \ -DBUILD_VINEYARD_TESTS=OFF \ @@ -699,7 +699,7 @@ install_basic_packages() { } ANALYTICAL_MACOS=("apache-arrow" "boost" "gflags" "glog" "open-mpi" "openssl@1.1" "protobuf" "grpc" "rapidjson" "msgpack-cxx" "librdkafka" "patchelf") -ANALYTICAL_UBUNTU=("libboost-all-dev" "libopenmpi-dev" "libgflags-dev" "libgoogle-glog-dev" "libprotobuf-dev" "libgrpc++-dev" "libmsgpack-dev" "librdkafka-dev" "protobuf-compiler-grpc" "rapidjson-dev") +ANALYTICAL_UBUNTU=("libopenmpi-dev" "libgflags-dev" "libgoogle-glog-dev" "libprotobuf-dev" "libgrpc++-dev" "libmsgpack-dev" "librdkafka-dev" "protobuf-compiler-grpc" "rapidjson-dev") ANALYTICAL_CENTOS_7=("librdkafka-devel" "msgpack-devel" "rapidjson-devel") ANALYTICAL_CENTOS_8=("${ANALYTICAL_CENTOS_7[@]}" "boost-devel" "gflags-devel" "glog-devel") @@ -732,6 +732,8 @@ install_analytical_dependencies() { ${SUDO} apt-get install -y libarrow-dev=${ARROW_VERSION}-1 libarrow-dataset-dev=${ARROW_VERSION}-1 libarrow-acero-dev=${ARROW_VERSION}-1 libparquet-dev=${ARROW_VERSION}-1 rm /tmp/apache-arrow-apt-source-latest-*.deb fi + # install boost >= 1.75 for leaf + install_boost else if [[ "${OS_VERSION}" -eq "7" ]]; then ${SUDO} yum install -y ${ANALYTICAL_CENTOS_7[*]} From e4e7c127ffaef43da566386a17d232d8286f3f22 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Thu, 5 Sep 2024 14:13:40 +0800 Subject: [PATCH 2/8] update --- k8s/internal/Makefile | 2 +- python/graphscope/gsctl/scripts/install_deps.sh | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index 5493c425c167..a3025067e5f7 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -43,7 +43,7 @@ GRAPHSCOPE_HOME ?= /usr/local INSTALL_PREFIX ?= /opt/graphscope VERSION ?= latest -VINEYARD_VERSION ?= v0.23.0 +VINEYARD_VERSION ?= v0.22.0 PROFILE ?= release CI ?= false diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index 33d0828fd4b0..cf5b5f6fac77 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -196,9 +196,7 @@ function set_to_cn_url() { function fetch_source() { local url=$1 local file=$2 - command="curl -fsSL -o \"${file}\" \"${url}/${file}\"" - info ${command} - eval ${command} + curl -fsSL -o "${file}" "${url}/${file}" } function download_and_untar() { From 32f2d1eec0dab6c5f3b40c7bfca5ae9797ffe39f Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Thu, 5 Sep 2024 16:28:41 +0800 Subject: [PATCH 3/8] Upgrade v6d_version to v0.24.2 for building wheel packaging --- k8s/internal/Makefile | 9 +++++---- python/graphscope/gsctl/scripts/install_deps.sh | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index a3025067e5f7..cd7fb8dbbe8b 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -1,6 +1,7 @@ -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -WORKING_DIR := $(dir $(MKFILE_PATH)) -SHORT_SHA := $(shell git rev-parse --short HEAD) +MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +WORKING_DIR := $(dir $(MKFILE_PATH)) +SHORT_SHA := $(shell git rev-parse --short HEAD) +VINEYARD_VERSION_FILE := ${WORKING_DIR}/../../V6D_VERSION .PHONY: graphscope-jupyter-manylinux2014-py3 graphscope-py3-package graphscope-client-py3-package .PHONY: graphscope-darwin-py3 graphscope-client-darwin-py3 @@ -43,7 +44,7 @@ GRAPHSCOPE_HOME ?= /usr/local INSTALL_PREFIX ?= /opt/graphscope VERSION ?= latest -VINEYARD_VERSION ?= v0.22.0 +VINEYARD_VERSION ?= $(shell cat $(VINEYARD_VERSION_FILE)) PROFILE ?= release CI ?= false diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index cf5b5f6fac77..909320896bc2 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -297,8 +297,10 @@ install_boost() { url=$(set_to_cn_url ${url}) download_and_untar "${url}" "${file}" "${directory}" pushd ${directory} || exit + # seastar needs filesystem program_options thread unit_test_framework + # interactive needs context regex date_time ./bootstrap.sh --prefix="${install_prefix}" \ - --with-libraries=system,filesystem,context,program_options,regex,thread,random,chrono,atomic,date_time + --with-libraries=system,filesystem,context,program_options,regex,thread,random,chrono,atomic,date_time,test ./b2 install link=shared runtime-link=shared variant=release threading=multi popd || exit popd || exit From 394f85c4c1492eb4f106558298a8404565656394 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Fri, 6 Sep 2024 11:43:29 +0800 Subject: [PATCH 4/8] update --- .github/workflows/interactive.yml | 10 +++++----- .../workflows/networkx-forward-algo-nightly.yml | 2 +- .../core/fragment/dynamic_fragment.h | 8 ++++++++ .../core/fragment/dynamic_projected_fragment.h | 16 ++++++++++++++++ python/setup_gsctl.py | 2 +- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 7c5610e28374..80a65d8934cf 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-20.04 if: ${{ github.repository == 'alibaba/GraphScope' }} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0 + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64 steps: - uses: actions/checkout@v4 @@ -353,7 +353,7 @@ jobs: test-cmake-options: runs-on: ubuntu-20.04 container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0 + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64 strategy: matrix: BUILD_TEST: [ON, OFF] @@ -373,7 +373,7 @@ jobs: test-AOCC-compilation: runs-on: ubuntu-20.04 container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0 + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64 steps: - uses: actions/checkout@v4 @@ -402,7 +402,7 @@ jobs: runs-on: ubuntu-20.04 if: ${{ github.repository == 'alibaba/GraphScope' }} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0 + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64 steps: - uses: actions/checkout@v4 @@ -553,4 +553,4 @@ jobs: cd ${GITHUB_WORKSPACE}/flex/build/ SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/tests/rt_mutable_graph/movie_schema_test.yaml BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/tests/rt_mutable_graph/movie_import_test.yaml - GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/ \ No newline at end of file + GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/ diff --git a/.github/workflows/networkx-forward-algo-nightly.yml b/.github/workflows/networkx-forward-algo-nightly.yml index f395b0a4a7a3..4fe4993ca5d4 100644 --- a/.github/workflows/networkx-forward-algo-nightly.yml +++ b/.github/workflows/networkx-forward-algo-nightly.yml @@ -17,7 +17,7 @@ jobs: run: shell: bash --noprofile --norc -eo pipefail {0} container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.22.0 + image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64 options: --shm-size 4096m diff --git a/analytical_engine/core/fragment/dynamic_fragment.h b/analytical_engine/core/fragment/dynamic_fragment.h index 862d55fe4cf4..65becac15e49 100644 --- a/analytical_engine/core/fragment/dynamic_fragment.h +++ b/analytical_engine/core/fragment/dynamic_fragment.h @@ -428,6 +428,14 @@ class DynamicFragment return res; } + inline size_t GetOutgoingEdgeNum() const override { + return oe_.head_edge_num() + is_selfloops_.count(); + } + + inline size_t GetIncomingEdgeNum() const override { + return ie_.head_edge_num() + is_selfloops_.count(); + } + using base_t::InnerVertices; using base_t::IsInnerVertex; using base_t::OuterVertices; diff --git a/analytical_engine/core/fragment/dynamic_projected_fragment.h b/analytical_engine/core/fragment/dynamic_projected_fragment.h index 242c54b68982..1e4829ceb323 100644 --- a/analytical_engine/core/fragment/dynamic_projected_fragment.h +++ b/analytical_engine/core/fragment/dynamic_projected_fragment.h @@ -479,6 +479,10 @@ class DynamicProjectedFragment { inline size_t GetEdgeNum() const { return fragment_->GetEdgeNum(); } + inline size_t GetOutgoingEdgeNum() const { return fragment_->GetOutgoingEdgeNum(); } + + inline size_t GetIncomingEdgeNum() const { return fragment_->GetIncomingEdgeNum(); } + inline bool IsInnerVertex(const vertex_t& v) const { return fragment_->IsInnerVertex(v); } @@ -639,14 +643,26 @@ class DynamicProjectedFragment { return fragment_->IEDests(v); } + inline size_t IEDestsSize() const { + return fragment_->IEDestsSize(); + } + inline grape::DestList OEDests(const vertex_t& v) const { return fragment_->OEDests(v); } + inline size_t OEDestsSize() const { + return fragment_->OEDestsSize(); + } + inline grape::DestList IOEDests(const vertex_t& v) const { return fragment_->IOEDests(v); } + inline size_t IOEDestsSize() const { + return fragment_->IOEDestsSize(); + } + inline const std::vector& MirrorVertices(fid_t fid) const { return fragment_->MirrorVertices(fid); } diff --git a/python/setup_gsctl.py b/python/setup_gsctl.py index 3d118445204f..8a918be4e18a 100644 --- a/python/setup_gsctl.py +++ b/python/setup_gsctl.py @@ -46,7 +46,7 @@ def parse_version(root, **kwargs): if platform.system() == "Linux" and platform.machine() == "aarch64" else "click >= 8.1.6" ), - "graphscope-flex >= 0.27.0", + "graphscope-flex >= 0.28.0", "treelib", "packaging", "pyyaml", From e906b6d782357d6f18a3c08210c2278b09130cd1 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Fri, 6 Sep 2024 11:47:07 +0800 Subject: [PATCH 5/8] update --- .github/workflows/k8s-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index aec2ad1476c6..d167e6a74004 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -666,7 +666,7 @@ jobs: if: false # No need to run this test for now needs: [build-gie-experimental] runs-on: [self-hosted, ubuntu2004] - if: ${{ github.repository == 'alibaba/GraphScope' }} + # if: ${{ github.repository == 'alibaba/GraphScope' }} steps: - uses: actions/checkout@v4 From b93e21d61432fd37d26fc4529abb133e0e167723 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Fri, 6 Sep 2024 11:51:59 +0800 Subject: [PATCH 6/8] update --- .../fragment/dynamic_projected_fragment.h | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/analytical_engine/core/fragment/dynamic_projected_fragment.h b/analytical_engine/core/fragment/dynamic_projected_fragment.h index 1e4829ceb323..e1dae6ee77ca 100644 --- a/analytical_engine/core/fragment/dynamic_projected_fragment.h +++ b/analytical_engine/core/fragment/dynamic_projected_fragment.h @@ -479,9 +479,13 @@ class DynamicProjectedFragment { inline size_t GetEdgeNum() const { return fragment_->GetEdgeNum(); } - inline size_t GetOutgoingEdgeNum() const { return fragment_->GetOutgoingEdgeNum(); } + inline size_t GetOutgoingEdgeNum() const { + return fragment_->GetOutgoingEdgeNum(); + } - inline size_t GetIncomingEdgeNum() const { return fragment_->GetIncomingEdgeNum(); } + inline size_t GetIncomingEdgeNum() const { + return fragment_->GetIncomingEdgeNum(); + } inline bool IsInnerVertex(const vertex_t& v) const { return fragment_->IsInnerVertex(v); @@ -643,25 +647,19 @@ class DynamicProjectedFragment { return fragment_->IEDests(v); } - inline size_t IEDestsSize() const { - return fragment_->IEDestsSize(); - } + inline size_t IEDestsSize() const { return fragment_->IEDestsSize(); } inline grape::DestList OEDests(const vertex_t& v) const { return fragment_->OEDests(v); } - inline size_t OEDestsSize() const { - return fragment_->OEDestsSize(); - } + inline size_t OEDestsSize() const { return fragment_->OEDestsSize(); } inline grape::DestList IOEDests(const vertex_t& v) const { return fragment_->IOEDests(v); } - inline size_t IOEDestsSize() const { - return fragment_->IOEDestsSize(); - } + inline size_t IOEDestsSize() const { return fragment_->IOEDestsSize(); } inline const std::vector& MirrorVertices(fid_t fid) const { return fragment_->MirrorVertices(fid); From 7ff627f4483d1320a57efc64a6e456e6330b13ea Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Fri, 6 Sep 2024 13:15:42 +0800 Subject: [PATCH 7/8] update --- analytical_engine/core/fragment/dynamic_fragment.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analytical_engine/core/fragment/dynamic_fragment.h b/analytical_engine/core/fragment/dynamic_fragment.h index 65becac15e49..ab5afc8bd64b 100644 --- a/analytical_engine/core/fragment/dynamic_fragment.h +++ b/analytical_engine/core/fragment/dynamic_fragment.h @@ -428,11 +428,11 @@ class DynamicFragment return res; } - inline size_t GetOutgoingEdgeNum() const override { + inline size_t GetOutgoingEdgeNum() const { return oe_.head_edge_num() + is_selfloops_.count(); } - inline size_t GetIncomingEdgeNum() const override { + inline size_t GetIncomingEdgeNum() const { return ie_.head_edge_num() + is_selfloops_.count(); } From 0e8ebb8dd54da52d7cdef1966697effb5a4c9a03 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Fri, 6 Sep 2024 14:02:13 +0800 Subject: [PATCH 8/8] update --- .../core/fragment/dynamic_projected_fragment.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/analytical_engine/core/fragment/dynamic_projected_fragment.h b/analytical_engine/core/fragment/dynamic_projected_fragment.h index e1dae6ee77ca..63d8acf1193b 100644 --- a/analytical_engine/core/fragment/dynamic_projected_fragment.h +++ b/analytical_engine/core/fragment/dynamic_projected_fragment.h @@ -803,6 +803,14 @@ class DynamicProjectedFragment { inline size_t GetEdgeNum() const { return fragment_->GetEdgeNum(); } + inline size_t GetOutgoingEdgeNum() const { + return fragment_->GetOutgoingEdgeNum(); + } + + inline size_t GetIncomingEdgeNum() const { + return fragment_->GetIncomingEdgeNum(); + } + inline bool IsInnerVertex(const vertex_t& v) const { return fragment_->IsInnerVertex(v); } @@ -948,14 +956,20 @@ class DynamicProjectedFragment { return fragment_->IEDests(v); } + inline size_t IEDestsSize() const { return fragment_->IEDestsSize(); } + inline grape::DestList OEDests(const vertex_t& v) const { return fragment_->OEDests(v); } + inline size_t OEDestsSize() const { return fragment_->OEDestsSize(); } + inline grape::DestList IOEDests(const vertex_t& v) const { return fragment_->IOEDests(v); } + inline size_t IOEDestsSize() const { return fragment_->IOEDestsSize(); } + inline const std::vector& MirrorVertices(fid_t fid) const { return fragment_->MirrorVertices(fid); }