From 354c20074f14e8c5881f4db47cd6f7e4764c897d Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Mon, 19 Aug 2024 16:30:42 +0800 Subject: [PATCH 1/7] feat: Reorganize gsctl --- .../workflows/build-graphscope-dev-images.yml | 181 +--- .../build-graphscope-dev-wheel-images.yml | 56 -- .../build-graphscope-wheels-macos.yml | 4 +- .github/workflows/flex-interactive-dummy.yml | 23 - .github/workflows/gae-dummy.yml | 23 - .github/workflows/gae.yml | 41 +- .github/workflows/gss-dummy.yml | 42 - .github/workflows/k8s-ci-dummy.yml | 115 --- .github/workflows/local-ci-dummy.yml | 106 --- V6D_VERSION | 1 + .../core/fragment/arrow_projected_fragment.h | 6 + k8s/Makefile | 41 +- .../manylinux/Dockerfile | 5 +- .../manylinux/Makefile | 9 +- k8s/dockerfiles/analytical.Dockerfile | 11 +- k8s/dockerfiles/coordinator.Dockerfile | 4 +- k8s/dockerfiles/flex-interactive.Dockerfile | 4 +- k8s/dockerfiles/graphlearn-torch.Dockerfile | 7 +- .../graphscope-dev-wheel.Dockerfile | 4 +- k8s/dockerfiles/graphscope-dev.Dockerfile | 5 +- k8s/dockerfiles/graphscope-store.Dockerfile | 4 +- .../interactive-experimental.Dockerfile | 4 +- k8s/dockerfiles/interactive.Dockerfile | 7 +- k8s/dockerfiles/learning.Dockerfile | 7 +- k8s/dockerfiles/manylinux2014.Dockerfile | 4 +- k8s/dockerfiles/vineyard-dev.Dockerfile | 4 +- k8s/dockerfiles/vineyard-runtime.Dockerfile | 4 +- python/graphscope/gsctl/V6D_VERSION | 1 + python/graphscope/gsctl/commands/__init__.py | 36 +- python/graphscope/gsctl/commands/dev.py | 338 +------ python/graphscope/gsctl/gsctl.py | 5 - python/graphscope/gsctl/scripts/__init__.py | 0 .../graphscope/gsctl/scripts/dev_command.sh | 41 - .../gsctl/scripts/format_command.sh | 103 --- python/graphscope/gsctl/scripts/initialize.sh | 23 - .../graphscope/gsctl/scripts/install_deps.sh | 872 ++++++++++++++++++ .../gsctl/scripts/install_deps_command.sh | 448 --------- python/graphscope/gsctl/scripts/lib/colors.sh | 42 - .../gsctl/scripts/lib/get_os_version.sh | 34 - .../lib/install_thirdparty_dependencies.sh | 472 ---------- .../gsctl/scripts/lib/install_vineyard.sh | 55 -- python/graphscope/gsctl/scripts/lib/log.sh | 20 - python/graphscope/gsctl/scripts/lib/util.sh | 64 -- .../graphscope/gsctl/scripts/make_command.sh | 107 --- .../gsctl/scripts/make_image_command.sh | 95 -- .../gsctl/scripts/spark_classpath_command.sh | 22 - .../graphscope/gsctl/scripts/test_command.sh | 172 ---- python/graphscope/gsctl/utils.py | 4 +- python/graphscope/gsctl/version.py | 12 +- python/setup_gsctl.py | 13 +- 50 files changed, 1062 insertions(+), 2639 deletions(-) delete mode 100644 .github/workflows/build-graphscope-dev-wheel-images.yml delete mode 100644 .github/workflows/flex-interactive-dummy.yml delete mode 100644 .github/workflows/gae-dummy.yml delete mode 100644 .github/workflows/gss-dummy.yml delete mode 100644 .github/workflows/k8s-ci-dummy.yml delete mode 100644 .github/workflows/local-ci-dummy.yml create mode 100644 V6D_VERSION create mode 120000 python/graphscope/gsctl/V6D_VERSION delete mode 100644 python/graphscope/gsctl/scripts/__init__.py delete mode 100644 python/graphscope/gsctl/scripts/dev_command.sh delete mode 100644 python/graphscope/gsctl/scripts/format_command.sh delete mode 100644 python/graphscope/gsctl/scripts/initialize.sh create mode 100755 python/graphscope/gsctl/scripts/install_deps.sh delete mode 100644 python/graphscope/gsctl/scripts/install_deps_command.sh delete mode 100644 python/graphscope/gsctl/scripts/lib/colors.sh delete mode 100644 python/graphscope/gsctl/scripts/lib/get_os_version.sh delete mode 100644 python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh delete mode 100644 python/graphscope/gsctl/scripts/lib/install_vineyard.sh delete mode 100644 python/graphscope/gsctl/scripts/lib/log.sh delete mode 100644 python/graphscope/gsctl/scripts/lib/util.sh delete mode 100644 python/graphscope/gsctl/scripts/make_command.sh delete mode 100644 python/graphscope/gsctl/scripts/make_image_command.sh delete mode 100644 python/graphscope/gsctl/scripts/spark_classpath_command.sh delete mode 100644 python/graphscope/gsctl/scripts/test_command.sh diff --git a/.github/workflows/build-graphscope-dev-images.yml b/.github/workflows/build-graphscope-dev-images.yml index 22eba205b978..749f1195f876 100644 --- a/.github/workflows/build-graphscope-dev-images.yml +++ b/.github/workflows/build-graphscope-dev-images.yml @@ -1,11 +1,10 @@ -name: Build GraphScope Development Images +name: Build GraphScope Dev Images # build images for: # 1) graphscope-dev: including all dependencies for graphscope development env. # 2) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine. # 3) vineyard-runtime: including all vineyard-related running dependencies. -# Note that: -# Due to security considerations, we cannot use self-hosts runner(aarch64) when we configured the secret on github. +# 4) `graphscope-dev:wheel`: including all dependencies for building graphscope wheel package on: workflow_dispatch: inputs: @@ -23,6 +22,11 @@ on: required: true default: true type: boolean + build_graphscope_dev_wheel: + description: 'Whether to build graphscope-dev:wheel image' + required: true + default: true + type: boolean pull_request: branches: - main @@ -38,8 +42,8 @@ env: REGISTRY: registry.cn-hongkong.aliyuncs.com jobs: - build-graphscope-dev-image-amd64: - if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request') + build-graphscope-dev-wheel-image-amd64: + if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev_wheel == 'true') || (github.event_name == 'pull_request') runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -48,15 +52,15 @@ jobs: - name: Build Image run: | - # build graphscope dev image with specified v6d version + # build wheel image with specified v6d's version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:-amd64 - make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION} + # graphscope/graphscope-dev:wheel--amd64 + make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version - make graphscope-dev + make dev-wheel fi - name: Release Image @@ -67,12 +71,12 @@ jobs: run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 + sudo docker tag graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 - build-graphscope-dev-image-arm64: - runs-on: [self-hosted, Linux, ARM64] - if: false + build-graphscope-dev-image-amd64: + if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request') + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 with: @@ -84,7 +88,7 @@ jobs: cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:-arm64 + # graphscope/graphscope-dev:-amd64 make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -99,28 +103,8 @@ jobs: run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - # graphscope-dev image - sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 - - manifest-push-graphscope-dev-image: - runs-on: ubuntu-20.04 - if: false - needs: [build-graphscope-dev-image-amd64, build-graphscope-dev-image-arm64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-amd64 \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-arm64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 build-vineyard-dev-image-amd64: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request') @@ -154,58 +138,6 @@ jobs: sudo docker tag graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 - build-vineyard-dev-image-arm64: - runs-on: [self-hosted, Linux, ARM64] - if: false - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - # build vineyard dev image with specified v6d version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-dev:-aarch64 - make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make vineyard-dev - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - # vineyard-dev image - sudo docker tag graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 - - manifest-push-vineyard-dev-image: - runs-on: ubuntu-20.04 - if: false - needs: [build-vineyard-dev-image-amd64, build-vineyard-dev-image-arm64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-amd64 \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-arm64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} - build-vineyard-runtime-image-amd64: # only trigger this step in 'workflow_dispatch' event, since the 'vineyard-dev' image isn't actually pushed in 'pull_request' if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request') @@ -239,74 +171,3 @@ jobs: VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} sudo docker tag graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 - - build-vineyard-runtime-image-arm64: - runs-on: [self-hosted, Linux, ARM64] - # only trigger this step in 'workflow_dispatch' event, - # since the 'vineyard-dev' image isn't actually pushed in 'pull_request' - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request') - if: false - - needs: [manifest-push-vineyard-dev-image] - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - # build vineyard dev image with specified v6d version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-runtime:-aarch64 - make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make vineyard-runtime - fi - - - name: Release Image - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # vineyard-runtime image - sudo docker tag graphscope/vineyard-runtime:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/vineyard-runtime:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} || true - - manifest-push-vineyard-runtime-image: - runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} - if: false - - needs: [build-vineyard-runtime-image-amd64, build-vineyard-runtime-image-arm64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-aarch64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }} diff --git a/.github/workflows/build-graphscope-dev-wheel-images.yml b/.github/workflows/build-graphscope-dev-wheel-images.yml deleted file mode 100644 index 791a88e4a1fa..000000000000 --- a/.github/workflows/build-graphscope-dev-wheel-images.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Build GraphScope Wheel Images - -# build `graphscope/graphscope-dev:wheel-{v6d_version}` image based on manylinux, -# including all dependencies for building graphscope wheel package. -on: - workflow_dispatch: - inputs: - v6d_version: - description: 'Version for Vineyard (v6d)' - required: true - default: 'main' - pull_request: - branches: - - main - paths: - - 'python/graphscope/gsctl/scripts/**' - - '.github/workflows/build-graphscope-dev-wheel-images.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - REGISTRY: registry.cn-hongkong.aliyuncs.com - -jobs: - build-wheel-image-amd64: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - # build wheel image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:wheel--amd64 - make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make dev-wheel - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - sudo docker tag graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml index bc7cb7362269..87969244d183 100644 --- a/.github/workflows/build-graphscope-wheels-macos.yml +++ b/.github/workflows/build-graphscope-wheels-macos.yml @@ -48,7 +48,7 @@ jobs: # Not sure when the first installation would fail, saying that the lock of python3.10 is taken # However the second trial would success. - python3 -m pip install click packaging + python3 -m pip install click python3 gsctl.py install-deps dev || true python3 gsctl.py install-deps dev echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env @@ -166,7 +166,7 @@ jobs: sudo mkdir /opt/graphscope sudo chown -R $(id -u):$(id -g) /opt/graphscope - python3 -m pip install click packaging + python3 -m pip install click python3 gsctl.py install-deps dev || true python3 gsctl.py install-deps dev diff --git a/.github/workflows/flex-interactive-dummy.yml b/.github/workflows/flex-interactive-dummy.yml deleted file mode 100644 index 1d05cffc3363..000000000000 --- a/.github/workflows/flex-interactive-dummy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Flex Interactive CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!flex/**' - - '!coordinator/gscoordinator/flex/**' - - '!python/graphscope/gsctl/**' - - '!.github/workflows/flex-interactive.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/gae-dummy.yml b/.github/workflows/gae-dummy.yml deleted file mode 100644 index b405f337efc3..000000000000 --- a/.github/workflows/gae-dummy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: GraphScope GAE CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!analytical_engine/**' - - '!.github/workflows/gae.yml' - - '!coordinator/gscoordinator/template/CMakeLists.template' # to verify whether java app can be successfully built when CMakeLists.template changes. - - 'analytical_engine/**.md' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - build-gae: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml index da8ad6ee82a6..7b338e5b0371 100644 --- a/.github/workflows/gae.yml +++ b/.github/workflows/gae.yml @@ -1,8 +1,6 @@ name: GraphScope GAE CI on: - # Trigger the workflow on push or pull request, - # but only for the main branch workflow_dispatch: push: branches: @@ -26,49 +24,41 @@ concurrency: cancel-in-progress: true jobs: - build-gae: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - container: - image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.22.0 + gae-test: + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - - name: Install latest vineyard - if: false + - name: Install dependencies run: | - . /home/graphscope/.graphscope_env - sudo pip3 install libclang - git clone --single-branch --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d - cd /tmp/v6d - git submodule update --init - mkdir -p build && cd build - cmake .. -DCMAKE_PREFIX_PATH=/opt/graphscope \ - -DCMAKE_INSTALL_PREFIX=/opt/graphscope \ - -DOPENSSL_ROOT_DIR=/opt/graphscope \ - -DUSE_EXTERNAL_ETCD_LIBS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_VINEYARD_TESTS=OFF - make -j$(nproc) - sudo make install + cd ${GITHUB_WORKSPACE} + python3 -m pip install --upgrade pip && python3 -m pip install click + python3 ./gsctl.py install-deps dev-analytical-java - name: Build env: RUN_JAVA_TESTS: ON run: | + source ${HOME}/.graphscope_env # default install to "/opt/graphscope" + cd ${GITHUB_WORKSPACE} make analytical-java BUILD_TEST=ON NUMPROC=1 sudo make analytical-java-install # also make coordinator and client for python test python3 -m pip install --upgrade pip setuptools make coordinator + - name: Setup tmate session + if: false + uses: mxschmitt/action-tmate@v3 + - name: Run Cpp Test env: - GRAPHSCOPE_HOME: /opt/graphscope GS_TEST_DIR: ${{ github.workspace }}/gstest RUN_JAVA_TESTS: ON run: | + source ${HOME}/.graphscope_env + # download dataset git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR} @@ -88,8 +78,9 @@ jobs: env: GS_TEST_DIR: ${{ github.workspace }}/gstest RUN_JAVA_TESTS: ON - GRAPHSCOPE_HOME: /opt/graphscope run: | + source ${HOME}/.graphscope_env + version=$(cat ${GITHUB_WORKSPACE}/VERSION) export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-${version}-shaded.jar diff --git a/.github/workflows/gss-dummy.yml b/.github/workflows/gss-dummy.yml deleted file mode 100644 index d309323ab8fb..000000000000 --- a/.github/workflows/gss-dummy.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: GraphScope Store CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!proto/**' - - '!interactive_engine/**' - - '!python/graphscope/client/**' - - '!charts/graphscope-store/**' - - '!.github/workflows/gss.yml' - - '!flex/openapi/openapi_coordinator.yaml' - - '!coordinator/gscoordinator/flex/**' - - '!python/graphscope/gsctl/**' - - 'interactive_engine/**.md' - - 'charts/graphscope-store/**.md' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GSS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store - -jobs: - gremlin-test: - # Require the host is able to run docker without sudo and - # can `ssh localhost` without password, which may need to - # be configured manually when a new self-hosted runner is added. - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - helm-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [gremlin-test] - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/k8s-ci-dummy.yml b/.github/workflows/k8s-ci-dummy.yml deleted file mode 100644 index 78857883c413..000000000000 --- a/.github/workflows/k8s-ci-dummy.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: GraphScope CI on Kubernetes (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!.github/workflows/k8s-ci.yml' - - '!Makefile' - - '!analytical_engine/**' - - '!charts/**' - - '!coordinator/**' - - '!interactive_engine/**' - - '!k8s/**' - - '!learning_engine/**' - - '!proto/**' - - '!python/**' - - '**.md' - - '**.rst' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - gae-python: ${{ steps.filter.outputs.gae-python }} - networkx: ${{ steps.filter.outputs.networkx }} - gie-function-test: ${{ steps.filter.outputs.gie-function-test }} - steps: - # For push it's necessary to checkout the code - - uses: actions/checkout@v4 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - base: main # Change detection against merge-base with main before push - filters: | - gae-python: - - 'proto/**' - - 'analytical_engine/**' - - 'python/graphscope/analytical/**' - - 'python/graphscope/client/**' - - 'python/graphscope/dataset/**' - - 'python/graphscope/deploy/**' - - 'python/graphscope/framework/**' - - 'python/graphscope/tests/unittest/**' - - 'coordinator/gscoordinator/**' - - '.github/workflows/ci.yml' - networkx: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/**' - gie-function-test: - - 'interactive_engine/**' - - 'python/graphscope/interactive/**' - - '.github/workflows/ci.yml' - - build-analytical: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-analytical-java: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-interactive: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-learning: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - # build-coordinator: - # runs-on: ubuntu-20.04 - # needs: [changes] - # if: ${{ github.repository == 'alibaba/GraphScope' }} - # steps: - # - run: 'echo "No action required" ' - - k8s-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [build-analytical, build-analytical-java, build-interactive, build-learning] - steps: - - run: 'echo "No action required" ' - - gie-test: - runs-on: ubuntu-20.04 - needs: [build-analytical, build-analytical-java, build-interactive, build-learning] - if: ${{ (needs.changes.outputs.gie-function-test == 'false' || github.ref == 'refs/heads/main') && github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/local-ci-dummy.yml b/.github/workflows/local-ci-dummy.yml deleted file mode 100644 index 0dc73235cc8c..000000000000 --- a/.github/workflows/local-ci-dummy.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: GraphScope CI on Local (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!.github/workflows/local-ci.yml' - - '!Makefile' - - '!analytical_engine/**' - - '!charts/**' - - '!coordinator/**' - - '!interactive_engine/**' - - '!k8s/**' - - '!learning_engine/**' - - '!proto/**' - - '!python/**' - - '**.md' - - '**.rst' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - gae-python: ${{ steps.filter.outputs.gae-python }} - networkx: ${{ steps.filter.outputs.networkx }} - gie-function-test: ${{ steps.filter.outputs.gie-function-test }} - steps: - # For push it's necessary to checkout the code - - uses: actions/checkout@v4 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - base: main # Change detection against merge-base with main before push - filters: | - gae-python: - - 'proto/**' - - 'analytical_engine/**' - - 'python/graphscope/analytical/**' - - 'python/graphscope/client/**' - - 'python/graphscope/dataset/**' - - 'python/graphscope/deploy/**' - - 'python/graphscope/framework/**' - - 'python/graphscope/tests/unittest/**' - - 'coordinator/gscoordinator/**' - - '.github/workflows/ci.yml' - networkx: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/**' - gie-function-test: - - 'interactive_engine/**' - - 'python/graphscope/interactive/**' - - '.github/workflows/ci.yml' - - build-wheels: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - mini-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [build-wheels] - steps: - - run: 'echo "No action required" ' - - python-unittest: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' - - networkx-basic-test: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' - - networkx-algo-and-generator-test: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' diff --git a/V6D_VERSION b/V6D_VERSION new file mode 100644 index 000000000000..6205e5bf3ac1 --- /dev/null +++ b/V6D_VERSION @@ -0,0 +1 @@ +v0.24.2 diff --git a/analytical_engine/core/fragment/arrow_projected_fragment.h b/analytical_engine/core/fragment/arrow_projected_fragment.h index dea3967279da..f9c750612cdb 100644 --- a/analytical_engine/core/fragment/arrow_projected_fragment.h +++ b/analytical_engine/core/fragment/arrow_projected_fragment.h @@ -1471,18 +1471,24 @@ class ArrowProjectedFragment return grape::DestList(idoffset_[offset], idoffset_[offset + 1]); } + inline size_t IEDestsSize() const { return idst_.size(); } + inline grape::DestList OEDests(const vertex_t& v) const { int64_t offset = vid_parser_.GetOffset(v.GetValue()); assert(offset < static_cast(ivnum_)); return grape::DestList(odoffset_[offset], odoffset_[offset + 1]); } + inline size_t OEDestsSize() const { return odst_.size(); } + inline grape::DestList IOEDests(const vertex_t& v) const { int64_t offset = vid_parser_.GetOffset(v.GetValue()); assert(offset < static_cast(ivnum_)); return grape::DestList(iodoffset_[offset], iodoffset_[offset + 1]); } + inline size_t IOEDestsSize() const { return iodst_.size(); } + inline std::shared_ptr GetVertexMap() { return vm_ptr_; } inline const std::shared_ptr GetVertexMap() const { diff --git a/k8s/Makefile b/k8s/Makefile index 511aea9ec5a4..54d9058fec40 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -1,6 +1,7 @@ -MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -WORKING_DIR := $(dir $(MKFILE_PATH)) -DOCKERFILES_DIR := $(WORKING_DIR)/dockerfiles +MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +WORKING_DIR := $(dir $(MKFILE_PATH)) +DOCKERFILES_DIR := $(WORKING_DIR)/dockerfiles +VINEYARD_VERSION_FILE := ${WORKING_DIR}/../V6D_VERSION ifeq ($(REGISTRY),) REGISTRY := registry.cn-hongkong.aliyuncs.com @@ -12,11 +13,7 @@ ARCH := $(subst x86_64,amd64,$(subst aarch64,arm64,$(PLATFORM))) VERSION ?= latest -VINEYARD_VERSION ?= v0.23.0 -# This is the version of builder base image in most cases, except for graphscope-dev -BUILDER_VERSION ?= $(VINEYARD_VERSION) -# This is the version of runtime base image -RUNTIME_VERSION ?= $(VINEYARD_VERSION) +VINEYARD_VERSION ?= $(shell cat $(VINEYARD_VERSION_FILE)) PROFILE ?= release CI ?= false # Flex Coordinator @@ -85,7 +82,7 @@ vineyard-runtime: docker build \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ -t graphscope/vineyard-runtime:${VINEYARD_VERSION}-${ARCH} \ -f $(DOCKERFILES_DIR)/vineyard-runtime.Dockerfile . @@ -95,7 +92,7 @@ coordinator: --target coordinator \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg CI=${CI} \ -t graphscope/coordinator:${VERSION} \ -f $(DOCKERFILES_DIR)/coordinator.Dockerfile . @@ -106,8 +103,7 @@ analytical: --target analytical \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ - --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg CI=${CI} \ -t graphscope/analytical:${VERSION} \ -f $(DOCKERFILES_DIR)/analytical.Dockerfile . @@ -118,8 +114,7 @@ analytical-java: --target analytical-java \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ - --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg CI=${CI} \ -t graphscope/analytical-java:${VERSION} \ -f $(DOCKERFILES_DIR)/analytical.Dockerfile . @@ -130,8 +125,7 @@ interactive-frontend: --target frontend \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ - --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg profile=$(PROFILE) \ --build-arg CI=${CI} \ -t graphscope/interactive-frontend:${VERSION} \ @@ -143,8 +137,7 @@ interactive-executor: --target executor \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ - --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg profile=$(PROFILE) \ --build-arg CI=${CI} \ -t graphscope/interactive-executor:${VERSION} \ @@ -157,7 +150,7 @@ interactive-experimental: --target experimental \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ -t graphscope/interactive-experimental:${VERSION} \ -f $(DOCKERFILES_DIR)/interactive-experimental.Dockerfile . @@ -168,7 +161,7 @@ flex-interactive: --target runtime \ --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg PLATFORM=${PLATFORM} \ --build-arg ENABLE_OPENTELMETRY=${ENABLE_OPENTELMETRY} \ --build-arg ARCH=${ARCH} \ @@ -182,8 +175,7 @@ learning: --target learning \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ - --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg CI=${CI} \ -t graphscope/learning:${VERSION} \ -f $(DOCKERFILES_DIR)/learning.Dockerfile . @@ -194,8 +186,7 @@ graphlearn-torch: --target graphlearn-torch \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ - --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg CI=${CI} \ -t graphscope/graphlearn-torch:${VERSION} \ -f $(DOCKERFILES_DIR)/graphlearn-torch.Dockerfile . @@ -205,7 +196,7 @@ graphscope-store: docker build \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg profile=${PROFILE} \ --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ -t graphscope/graphscope-store:${VERSION} \ diff --git a/k8s/actions-runner-controller/manylinux/Dockerfile b/k8s/actions-runner-controller/manylinux/Dockerfile index 5c3ce0558cc3..87d0fa243388 100644 --- a/k8s/actions-runner-controller/manylinux/Dockerfile +++ b/k8s/actions-runner-controller/manylinux/Dockerfile @@ -1,6 +1,7 @@ ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION +ARG ARCH=amd64 +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH ARG TARGETPLATFORM ARG RUNNER_VERSION=2.287.1 diff --git a/k8s/actions-runner-controller/manylinux/Makefile b/k8s/actions-runner-controller/manylinux/Makefile index 9728c6fbaacf..8e687c77485e 100644 --- a/k8s/actions-runner-controller/manylinux/Makefile +++ b/k8s/actions-runner-controller/manylinux/Makefile @@ -1,3 +1,7 @@ +MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +WORKING_DIR := $(dir $(MKFILE_PATH)) +VINEYARD_VERSION_FILE := ${WORKING_DIR}/../../../V6D_VERSION + # it's ok to use registry-vpc here as building the runner image for CI # is only for internal usage and usually happens on ECS. ifeq ($(REGISTRY),) @@ -12,8 +16,7 @@ TARGETPLATFORM ?= $(shell arch) RUNNER_VERSION ?= 2.318.0 DOCKER_VERSION ?= 24.0.7 -VINEYARD_VERSION ?= v0.22.0 -BUILDER_VERSION ?= $(VINEYARD_VERSION) +VINEYARD_VERSION ?= $(shell cat $(VINEYARD_VERSION_FILE)) # default list of platforms for which multiarch image is built ifeq (${PLATFORMS}, ) @@ -26,5 +29,5 @@ build: --build-arg TARGETPLATFORM=${TARGETPLATFORM} \ --build-arg RUNNER_VERSION=${RUNNER_VERSION} \ --build-arg DOCKER_VERSION=${DOCKER_VERSION} \ - --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ -t ${NAME}:${TAG} . diff --git a/k8s/dockerfiles/analytical.Dockerfile b/k8s/dockerfiles/analytical.Dockerfile index c2cfc8a248b9..e5eea7bd383f 100644 --- a/k8s/dockerfiles/analytical.Dockerfile +++ b/k8s/dockerfiles/analytical.Dockerfile @@ -2,10 +2,9 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -ARG RUNTIME_VERSION=latest +ARG VINEYARD_VERSION=latest ############### BUILDER: ANALYTICAL ####################### -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder ARG CI=false @@ -27,7 +26,7 @@ RUN cd /home/graphscope/GraphScope/ && \ fi ############### RUNTIME: ANALYTICAL ####################### -FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION-$ARCH AS analytical +FROM $REGISTRY/graphscope/vineyard-dev:$VINEYARD_VERSION-$ARCH AS analytical ENV GRAPHSCOPE_HOME=/opt/graphscope ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GRAPHSCOPE_HOME/lib @@ -59,7 +58,7 @@ COPY ./k8s/dockerfiles/entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ############### BUILDER: ANALYTICAL-JAVA ####################### -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder-java +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder-java COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope @@ -82,7 +81,7 @@ RUN cd /home/graphscope/GraphScope/ && \ FROM vineyardcloudnative/manylinux-llvm:2014-11.0.0 AS llvm -FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION-$ARCH AS analytical-java +FROM $REGISTRY/graphscope/vineyard-dev:$VINEYARD_VERSION-$ARCH AS analytical-java COPY --from=llvm /opt/llvm11.0.0 /opt/llvm11 ENV LLVM11_HOME=/opt/llvm11 ENV LIBCLANG_PATH=$LLVM11_HOME/lib LLVM_CONFIG_PATH=$LLVM11_HOME/bin/llvm-config diff --git a/k8s/dockerfiles/coordinator.Dockerfile b/k8s/dockerfiles/coordinator.Dockerfile index 2eff6059a2cf..6e583dd73fdf 100644 --- a/k8s/dockerfiles/coordinator.Dockerfile +++ b/k8s/dockerfiles/coordinator.Dockerfile @@ -2,8 +2,8 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder ARG CI=false diff --git a/k8s/dockerfiles/flex-interactive.Dockerfile b/k8s/dockerfiles/flex-interactive.Dockerfile index f97250335559..f49ca6de8c1d 100644 --- a/k8s/dockerfiles/flex-interactive.Dockerfile +++ b/k8s/dockerfiles/flex-interactive.Dockerfile @@ -3,8 +3,8 @@ ARG PLATFORM=x86_64 ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder ARG ENABLE_COORDINATOR="false" ARG OPTIMIZE_FOR_HOST=OFF ARG ENABLE_OPENTELMETRY=false diff --git a/k8s/dockerfiles/graphlearn-torch.Dockerfile b/k8s/dockerfiles/graphlearn-torch.Dockerfile index 970eebc2282e..949714fa0350 100644 --- a/k8s/dockerfiles/graphlearn-torch.Dockerfile +++ b/k8s/dockerfiles/graphlearn-torch.Dockerfile @@ -2,9 +2,8 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=v0.21.3 -ARG RUNTIME_VERSION=v0.21.3 -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope @@ -25,7 +24,7 @@ RUN cd /home/graphscope/GraphScope/; \ cp dist/*.whl /home/graphscope/install/ ############### RUNTIME: GLE ####################### -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS graphlearn-torch +FROM $REGISTRY/graphscope/vineyard-runtime:$VINEYARD_VERSION-$ARCH AS graphlearn-torch RUN sudo apt-get update -y && \ sudo apt-get install -y python3-pip && \ diff --git a/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile b/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile index 1a58a42839f8..4383c3a1bc0b 100644 --- a/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile @@ -61,8 +61,8 @@ WORKDIR /home/graphscope COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main RUN cd /home/graphscope/GraphScope && \ - python3 -m pip install click packaging --user && \ - python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ + python3 -m pip install click --user && \ + python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION && \ sudo rm -rf /home/graphscope/GraphScope && \ sudo yum clean all -y && \ sudo rm -fr /var/cache/yum diff --git a/k8s/dockerfiles/graphscope-dev.Dockerfile b/k8s/dockerfiles/graphscope-dev.Dockerfile index 1c13a487033d..3cd843e36eb4 100644 --- a/k8s/dockerfiles/graphscope-dev.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev.Dockerfile @@ -29,9 +29,10 @@ WORKDIR /home/graphscope COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main + RUN cd /home/graphscope/GraphScope && \ - python3 -m pip install click packaging && \ - python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn -j $(nproc) && \ + python3 -m pip install click && \ + python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn && \ cd /home/graphscope && \ rm -fr GraphScope diff --git a/k8s/dockerfiles/graphscope-store.Dockerfile b/k8s/dockerfiles/graphscope-store.Dockerfile index 19b694b3f085..b055e3bd5628 100644 --- a/k8s/dockerfiles/graphscope-store.Dockerfile +++ b/k8s/dockerfiles/graphscope-store.Dockerfile @@ -1,7 +1,7 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH as builder +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH as builder ARG CI=false ARG ENABLE_COORDINATOR=false diff --git a/k8s/dockerfiles/interactive-experimental.Dockerfile b/k8s/dockerfiles/interactive-experimental.Dockerfile index 478fa3bf60e5..c9eda8f21c68 100644 --- a/k8s/dockerfiles/interactive-experimental.Dockerfile +++ b/k8s/dockerfiles/interactive-experimental.Dockerfile @@ -2,8 +2,8 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope diff --git a/k8s/dockerfiles/interactive.Dockerfile b/k8s/dockerfiles/interactive.Dockerfile index 5ab58e8bd340..04132f13763e 100644 --- a/k8s/dockerfiles/interactive.Dockerfile +++ b/k8s/dockerfiles/interactive.Dockerfile @@ -2,9 +2,8 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -ARG RUNTIME_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder ARG CI=false @@ -54,7 +53,7 @@ WORKDIR /home/graphscope ############### RUNTIME: executor ####################### FROM registry.cn-hongkong.aliyuncs.com/graphscope/manylinux2014:$ARCH AS ext -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS executor +FROM $REGISTRY/graphscope/vineyard-runtime:$VINEYARD_VERSION-$ARCH AS executor ENV RUST_BACKTRACE=1 diff --git a/k8s/dockerfiles/learning.Dockerfile b/k8s/dockerfiles/learning.Dockerfile index 20b55f08fd67..21c019d2f524 100644 --- a/k8s/dockerfiles/learning.Dockerfile +++ b/k8s/dockerfiles/learning.Dockerfile @@ -2,9 +2,8 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -ARG RUNTIME_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION AS builder ARG CI=false @@ -30,7 +29,7 @@ RUN cd /home/graphscope/GraphScope/ && \ fi ############### RUNTIME: GLE ####################### -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS learning +FROM $REGISTRY/graphscope/vineyard-runtime:$VINEYARD_VERSION-$ARCH AS learning RUN sudo apt-get update -y && \ sudo apt-get install -y python3-pip && \ diff --git a/k8s/dockerfiles/manylinux2014.Dockerfile b/k8s/dockerfiles/manylinux2014.Dockerfile index fe2e85349658..876cd4bbb519 100644 --- a/k8s/dockerfiles/manylinux2014.Dockerfile +++ b/k8s/dockerfiles/manylinux2014.Dockerfile @@ -21,8 +21,8 @@ ENV LANG=en_US.utf-8 COPY . /root/GraphScope RUN cd /root/GraphScope && \ - python3 -m pip install click packaging && \ - python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ + python3 -m pip install click && \ + python3 gsctl.py install-deps dev-analytical --cn --no-v6d && \ rm -fr /root/GraphScope # install hadoop for processing hadoop data source diff --git a/k8s/dockerfiles/vineyard-dev.Dockerfile b/k8s/dockerfiles/vineyard-dev.Dockerfile index a68df0989273..2cca4427edb4 100644 --- a/k8s/dockerfiles/vineyard-dev.Dockerfile +++ b/k8s/dockerfiles/vineyard-dev.Dockerfile @@ -43,8 +43,8 @@ COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main RUN sudo chmod a+wrx /tmp && \ cd /home/graphscope/GraphScope && \ - python3 -m pip install click packaging && \ - python3 gsctl.py install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ + python3 -m pip install click && \ + python3 gsctl.py install-deps dev-analytical --v6d-version=$VINEYARD_VERSION && \ cd /home/graphscope && sudo rm -rf /home/graphscope/GraphScope RUN python3 -m pip --no-cache install pyyaml --user diff --git a/k8s/dockerfiles/vineyard-runtime.Dockerfile b/k8s/dockerfiles/vineyard-runtime.Dockerfile index 7bcd791fe762..4e9abf8a4e3c 100644 --- a/k8s/dockerfiles/vineyard-runtime.Dockerfile +++ b/k8s/dockerfiles/vineyard-runtime.Dockerfile @@ -3,8 +3,8 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/vineyard-dev:$BUILDER_VERSION-$ARCH AS builder +ARG VINEYARD_VERSION=latest +FROM $REGISTRY/graphscope/vineyard-dev:$VINEYARD_VERSION-$ARCH AS builder FROM ubuntu:22.04 AS runtime diff --git a/python/graphscope/gsctl/V6D_VERSION b/python/graphscope/gsctl/V6D_VERSION new file mode 120000 index 000000000000..ba52b47717e8 --- /dev/null +++ b/python/graphscope/gsctl/V6D_VERSION @@ -0,0 +1 @@ +../../../V6D_VERSION \ No newline at end of file diff --git a/python/graphscope/gsctl/commands/__init__.py b/python/graphscope/gsctl/commands/__init__.py index 516f882ea268..42c62789b105 100644 --- a/python/graphscope/gsctl/commands/__init__.py +++ b/python/graphscope/gsctl/commands/__init__.py @@ -33,6 +33,24 @@ from graphscope.gsctl.utils import err from graphscope.gsctl.utils import info +usage_message = """ +Currently, `gsctl` hasn't connect to any service. Try to deploy a GraphScope +instance with: + + gsctl instance deploy --type interactive + +See more detailed information at https://graphscope.io/docs/utilities/gs. +""" + +interactive_usage_message = """ +Currently in the global scope of Interactive instance, run `gsctl ls -l` to +display all resources in database and switch to a specific graph scope with: + + gsctl use GRAPH + +See more detailed information at https://graphscope.io/docs/utilities/gs. +""" + def is_interactive_mode(flex): return ( @@ -60,13 +78,7 @@ def get_command_collection(context: Context): if context is None: if len(sys.argv) == 1: info(logo, fg="green", bold=True) - click.secho("Currently, gsctl hasn't connect to any service.", fg="yellow") - message = """ -you can use gsctl as an utility script. -Or you can connect to a launched GraphScopoe service by `gsctl connect --coordinator-endpoint
`. -See more detailed information at https://graphscope.io/docs/utilities/gs. - """ - info(message) + info(usage_message, fg="red") return commands if context.is_expired(): @@ -101,10 +113,7 @@ def get_command_collection(context: Context): if is_interactive_mode(context.flex): if context.context == "global": if len(sys.argv) < 2 or sys.argv[1] != "use": - info("Using GLOBAL.", fg="green", bold=True) - info( - "Run `gsctl use GRAPH ` to switch to a specific graph context.\n" - ) + info(interactive_usage_message, fg="green") commands = click.CommandCollection(sources=[common, interactive]) else: if len(sys.argv) < 2 or sys.argv[1] != "use": @@ -113,7 +122,10 @@ def get_command_collection(context: Context): fg="green", bold=True, ) - info("Run `gsctl use GLOBAL` to switch back to GLOBAL context.\n") + info( + "Run `gsctl use GLOBAL` to switch back to GLOBAL context.\n", + fg="green", + ) commands = click.CommandCollection(sources=[common, interactive_graph]) elif is_insight_mode(context.flex): if context.context == "global": diff --git a/python/graphscope/gsctl/commands/dev.py b/python/graphscope/gsctl/commands/dev.py index 8138ebb14b62..8744b46f1df3 100644 --- a/python/graphscope/gsctl/commands/dev.py +++ b/python/graphscope/gsctl/commands/dev.py @@ -24,28 +24,27 @@ import sys import click -from packaging import version +# Make sure this file doesn't depend on any graphscope directories for +# installing dependencies, so we can't `import graphscope` here. version_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "VERSION" ) - with open(version_file_path, "r", encoding="utf-8") as fp: - sv = version.parse(fp.read().strip()) - __is_prerelease__ = sv.is_prerelease - __version__ = str(sv) + __version__ = fp.read().strip() +v6d_version_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "V6D_VERSION" +) +with open(v6d_version_file_path, "r", encoding="utf-8") as fp: + __v6d_version__ = fp.read().strip() # Interactive docker container config INTERACTIVE_DOCKER_CONTAINER_NAME = "gs-interactive-instance" INTERACTIVE_DOCKER_CONTAINER_LABEL = "flex=interactive" - scripts_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "scripts") -install_deps_script = os.path.join(scripts_dir, "install_deps_command.sh") -make_script = os.path.join(scripts_dir, "make_command.sh") -make_image_script = os.path.join(scripts_dir, "make_image_command.sh") -test_script = os.path.join(scripts_dir, "test_command.sh") +install_deps_script = os.path.join(scripts_dir, "install_deps.sh") default_graphscope_repo_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "..", @@ -57,14 +56,18 @@ def run_shell_cmd(cmd, workingdir): """wrapper function to run a shell command/scripts.""" - click.echo(f"run a shell command on cwd={workingdir}. \ncmd=\"{' '.join(cmd)}\"") + click.secho(f"run a shell command on cwd={workingdir}. \ncmd=\"{' '.join(cmd)}\"") proc = subprocess.Popen( - cmd, cwd=workingdir, env=os.environ.copy(), stdout=subprocess.PIPE + cmd, + cwd=workingdir, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, ) for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): - print(line.rstrip()) + click.secho(line.rstrip()) proc.wait() - sys.exit(proc.returncode) + return proc.returncode @click.group() @@ -81,7 +84,10 @@ def flexbuild(): @cli.group() def instance(): - """Deploy, destroy Interactive instance.""" + """Deploy, destroy a Flex instance. + + The `instance` subcommand is responsible for managing the Flex Instances. + """ pass @@ -109,7 +115,7 @@ def insight(app, graphscope_repo): ) return cmd = ["make", "graphscope-store", "ENABLE_COORDINATOR=true"] - run_shell_cmd(cmd, os.path.join(graphscope_repo, insight_build_dir)) + sys.exit(run_shell_cmd(cmd, os.path.join(graphscope_repo, insight_build_dir))) @flexbuild.command() @@ -132,13 +138,13 @@ def interactive(app, graphscope_repo): if not os.path.exists(interactive_build_dir) or not os.path.isdir( interactive_build_dir ): + click.secho("[FAILED] ", nl=False, fg="red", bold=True) click.secho( f"No such file or directory {interactive_build_dir}, try --graphscope-repo param.", - fg="red", ) return cmd = ["make", "flex-interactive", "ENABLE_COORDINATOR=true"] - run_shell_cmd(cmd, os.path.join(graphscope_repo, interactive_build_dir)) + sys.exit(run_shell_cmd(cmd, os.path.join(graphscope_repo, interactive_build_dir))) @instance.command @@ -215,7 +221,7 @@ def deploy( cypher_port, gremlin_port, ): # noqa: F811 - """Deploy Flex Interactive instance""" + """Deploy a GraphScope Flex instance""" cmd = [] if type == "interactive": cmd = [ @@ -239,12 +245,8 @@ def deploy( cmd.extend(["-p", f"{gremlin_port}:8182"]) image = f"{image_registry}/{type}:{image_tag}" cmd.extend([image, "--enable-coordinator"]) - click.secho("Run command: {0}".format(" ".join(cmd))) - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode == 0: - click.secho("[SUCCESS] ", nl=False, fg="green", bold=True) - click.secho(result.stdout, bold=False) - + returncode = run_shell_cmd(cmd, os.getcwd()) + if returncode == 0: message = f""" Coordinator is listening on {coordinator_port} port, you can connect to coordinator by: gsctl connect --coordinator-endpoint http://127.0.0.1:{coordinator_port} @@ -273,10 +275,8 @@ def deploy( Apart from interactive sdk, you can also use neo4j native tools(like cypher-shell) to connect to cypher endpoint, and gremlin console to connect to gremlin endpoint. """ - click.secho(message, bold=False) - else: - click.secho("[FAILED] ", nl=False, fg="red", bold=True) - click.secho(result.stderr, bold=False) + click.secho("[SUCCESS] ", nl=False, fg="green", bold=True) + click.secho(message, bold=False, fg="blue") @instance.command @@ -304,7 +304,6 @@ def destroy(type, container_name): "-f", container_name, ] - click.secho("Run command: {0}".format(" ".join(cmd))) result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode == 0: click.secho("[SUCCESS] ", nl=False, fg="green", bold=True) @@ -341,21 +340,24 @@ def status(type): click.secho(result.stderr, bold=False) -@click.command() +@cli.command() @click.argument( "type", type=click.Choice( - ["dev", "client"], + [ + "dev", + "dev-analytical", + "dev-analytical-java", + "dev-interactive", + "dev-learning", + ], case_sensitive=False, ), required=True, ) @click.option( "--graphscope-repo", - envvar="GRAPHSCOPE_REPO", - type=click.Path(), - default=os.path.abspath("."), - show_default=True, + required=False, help="GraphScope code repo location.", ) @click.option( @@ -369,50 +371,26 @@ def status(type): type=click.Path(), default="/opt/graphscope", show_default=True, - help="Install built binaries to customized location.", -) -@click.option( - "--from-local", - type=click.Path(), - default="/tmp/gs-local-deps", - show_default=True, - help="""Find raw dependencies of GraphScope from a local directory. The raw - dependencies would then be built and installed to [prefix]. If the directory - is empty or not exists, dependency files would be downloaded to [directory].""", + help="Specify the directory on disk to which the file will be installed", ) @click.option( "--v6d-version", - default="main", + default=__v6d_version__, show_default=True, - help="v6d version to clone.", -) -@click.option( - "-j", - "--jobs", - default="2", - help="Concurrent jobs in building, i.e., -j argument passed to make.", -) -@click.option( - "--for-analytical", - is_flag=True, - default=False, - help="Only install analytical engine dependencies.", + help="vineyard version", ) @click.option( "--no-v6d", is_flag=True, default=False, - help="Do not install v6d, for build base docker images, could only be used with '--for-analytical'", + help="Do not install vineyard, could be used with analytical type", ) def install_deps( type, graphscope_repo, cn, install_prefix, - from_local, v6d_version, - jobs, - for_analytical, no_v6d, ): """Install dependencies for building GraphScope.""" @@ -420,244 +398,18 @@ def install_deps( "bash", "-e", install_deps_script, - "-t", type, - "-i", + "--install-prefix", install_prefix, - "-d", - from_local, - "-v", + "--v6d-version", str(v6d_version), - "-j", - str(jobs), ] - if for_analytical: - cmd.append("--for-analytical") if no_v6d: - if not for_analytical: - # could only be used with '--for-analytical' - raise RuntimeError("Missing --for-analytical with --no-v6d parameter") cmd.append("--no-v6d") if cn: cmd.append("--cn") - run_shell_cmd(cmd, graphscope_repo) - - -@click.command() -@click.argument( - "component", - type=click.Choice( - [ - "interactive", - "interactive-install", - "analytical", - "analytical-java-install", - "analytical-install", - "learning", - "learning-install", - "coordinator", - "client", - "clean", - "all", - ], - case_sensitive=False, - ), - required=False, -) -@click.option( - "--graphscope-repo", - envvar="GRAPHSCOPE_REPO", - type=click.Path(), - default=os.path.abspath("."), - show_default=True, - help="GraphScope code repo location.", -) -@click.option( - "--install-prefix", - type=click.Path(), - default="/opt/graphscope", - show_default=True, - help="Install built binaries to customized location.", -) -@click.option( - "--storage-type", - default="default", - help="Make gie with specified storage type.", -) -def make(component, graphscope_repo, install_prefix, storage_type): - """Build executive binaries of COMPONENT. If not given a specific component, build all. - \f - TODO: maybe without make? - """ - click.secho( - "Before making artifacts, please manually source ENVs from ~/.graphscope_env.", - fg="yellow", - ) - click.secho( - f"Begin the make command, to build components [{component}] of GraphScope, with repo = {graphscope_repo}", - fg="green", - ) - if component is None: - component = "all" - - cmd = [ - "bash", - "-e", - make_script, - "-c", - component, - "-i", - install_prefix, - "-s", - storage_type, - ] - run_shell_cmd(cmd, graphscope_repo) - - -@click.command() -@click.argument( - "component", - type=click.Choice( - [ - "all", - "graphscope-dev", - "coordinator", - "analytical", - "analytical-java", - "interactive", - "interactive-frontend", - "interactive-executor", - "learning,", - "vineyard-dev", - "vineyard-runtime", - "manylinux2014-ext", - ], - case_sensitive=False, - ), - required=False, -) -@click.option( - "--graphscope-repo", - envvar="GRAPHSCOPE_REPO", - type=click.Path(), - default=os.path.abspath("."), - show_default=True, - help="GraphScope code repo location.", -) -@click.option( - "--tag", - default="latest", - show_default=True, - help="image tag name to build", -) -@click.option( - "--registry", - default="registry.cn-hongkong.aliyuncs.com", - show_default=True, - help="registry name", -) -def make_image(component, graphscope_repo, registry, tag): - """Make docker images from source code for deployment. - \f - TODO: fulfill this. - """ - if component is None: - component = "all" - - cmd = ["bash", "-e", make_image_script, "-c", component, "-r", registry, "-t", tag] - run_shell_cmd(cmd, graphscope_repo) - - -@click.command() -@click.argument( - "type", - type=click.Choice( - [ - "analytical", - "analytical-java", - "interactive", - "learning", - "local-e2e", - "k8s-e2e", - "groot", - ], - case_sensitive=False, - ), - required=False, -) -@click.option( - "--graphscope-repo", - envvar="GRAPHSCOPE_REPO", - type=click.Path(), - default=os.path.abspath("."), - show_default=True, - help="GraphScope code repo location.", -) -@click.option( - "--testdata", - type=click.Path(), - default="/tmp/gstest", - show_default=True, - help="""assign a custom test data location. This could be cloned from - https://github.com/graphscope/gstest""", -) -@click.option( - "--local", - is_flag=True, - default=False, - help="Run local tests", -) -@click.option( - "--storage-type", - default="default", - show_default=True, - help="test gie with specified storage type", -) -@click.option( - "--k8s", - is_flag=True, - default=False, - help="Run local tests", -) -@click.option( - "--nx", - is_flag=True, - default=False, - help="Run nx tests", -) -def test(type, graphscope_repo, testdata, local, storage_type, k8s, nx): - """Trigger tests on built artifacts. - - \f - TODO: fulfill this.""" - click.secho(f"graphscope_repo = {graphscope_repo}", fg="green") - click.echo("test") - if type is None: - type = "" - cmd = [ - "bash", - "-e", - test_script, - "-t", - type, - "-d", - testdata, - "-l", - str(local), - "-s", - storage_type, - "-k", - str(k8s), - "-n", - str(nx), - ] - run_shell_cmd(cmd, graphscope_repo) - + sys.exit(run_shell_cmd(cmd, graphscope_repo)) -cli.add_command(install_deps) -cli.add_command(make) -cli.add_command(make_image) -cli.add_command(test) if __name__ == "__main__": cli() diff --git a/python/graphscope/gsctl/gsctl.py b/python/graphscope/gsctl/gsctl.py index 72843c2c0d11..13c3eb9f8a0b 100644 --- a/python/graphscope/gsctl/gsctl.py +++ b/python/graphscope/gsctl/gsctl.py @@ -21,11 +21,6 @@ import click -if "site-packages" not in os.path.dirname(os.path.realpath(__file__)): - sys.path.insert( - 0, os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "..") - ) - try: import graphscope from graphscope.gsctl.commands import get_command_collection diff --git a/python/graphscope/gsctl/scripts/__init__.py b/python/graphscope/gsctl/scripts/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/python/graphscope/gsctl/scripts/dev_command.sh b/python/graphscope/gsctl/scripts/dev_command.sh deleted file mode 100644 index 6e137d2f9801..000000000000 --- a/python/graphscope/gsctl/scripts/dev_command.sh +++ /dev/null @@ -1,41 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh - - -while getopts ":l:" opt; do - case $opt in - l) - local="$OPTARG" - ;; - \?) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - - -mount_option="" - -if [[ -n $local ]]; then - echo "Opened a new container with $local mounted to /home/graphscope/graphscope." - mount_option="--mount type=bind,source=${local},target=/home/graphscope/graphscope" -else - echo "No local directory assigned, open a new container without mounting local directory." -fi - -# docker pull graphscope/graphscope-dev -REGISTRY="registry.cn-hongkong.aliyuncs.com" -docker run \ - -it \ - ${mount_option} \ - ${REGISTRY}/graphscope/graphscope-dev:latest diff --git a/python/graphscope/gsctl/scripts/format_command.sh b/python/graphscope/gsctl/scripts/format_command.sh deleted file mode 100644 index bfba0b2cfa68..000000000000 --- a/python/graphscope/gsctl/scripts/format_command.sh +++ /dev/null @@ -1,103 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh - - -echo "# this file is located in 'src/format_command.sh'" -echo "# code for 'gs format' goes here" -echo "# you can edit it freely and regenerate (it will not be overwritten)" - -while getopts ":l:" opt; do - case $opt in - l) - lang="$OPTARG" - ;; - \?) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - - - -GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" - -function format_cpp { - # Check if clang-format is installed - if ! [ -x "$(command -v clang-format)" ]; then - echo 'Downloading clang-format.' >&2 - curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output ${GRAPHSCOPE_HOME}/bin/clang-format - chmod +x ${GRAPHSCOPE_HOME}/clang-format - export PATH="${GRAPHSCOPE_HOME}/bin:${PATH}" - fi - # find all relevant files - pushd "${GS_SOURCE_DIR}"/analytical_engine || exit - files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) - - # run format - clang-format -i --style=file $(echo $files) - popd || exit -} - -function lint_cpp { - # use cpplint.py for static analysis - pushd "${GS_SOURCE_DIR}"/analytical_engine || exit - files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) - - ./misc/cpplint.py $(echo $files) - popd || exit -} - -function format_java { - jarfile=google-java-format-1.13.0-all-deps.jar - if [[ ! -f ${jarfile} ]]; then - wget https://github.com/google/google-java-format/releases/download/v1.13.0/${jarfile} - fi - # run formatter in-place - java -jar ${jarfile} --aosp --skip-javadoc-formatting -i $(git ls-files *.java) - -} - -function format_python { - # Install dependency - if ! [ -x "$(command -v black)" ]; then - pip3 install -r ${GS_SOURCE_DIR}/coordinator/requirements-dev.txt --user - fi - pushd python || exit - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - popd || exit - pushd coordinator || exit - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - popd || exit -} - -function format_rust { - # Use cargo fmt for formatting checks - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/assembly/groot - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/assembly/v6d - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/common/dyn_type/ - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/engine/pegasus/ - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/ir/ - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/store/ - cargo +nightly fmt -- --check -} - -format_"${lang}" diff --git a/python/graphscope/gsctl/scripts/initialize.sh b/python/graphscope/gsctl/scripts/initialize.sh deleted file mode 100644 index d3a8414bca95..000000000000 --- a/python/graphscope/gsctl/scripts/initialize.sh +++ /dev/null @@ -1,23 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh - - -## Code here runs inside the initialize() function -## Use it for anything that you need to run before any other function, like -## setting environment variables: -## CONFIG_FILE=settings.ini -## -## Feel free to empty (but not delete) this file. - -bash_source_dir="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" - -if [ -f "$HOME/.graphscope_env" ]; then - source $HOME/.graphscope_env -fi - -log "Read the env: GRAPHSCOPE_HOME=${GRAPHSCOPE_HOME:-}" diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh new file mode 100755 index 000000000000..d0bcaa20c70e --- /dev/null +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -0,0 +1,872 @@ +#!/bin/bash +set -e + +# blue +info() { + printf "\e[34m%b\e[0m\n" "$*" +} + +# red +err() { + printf "\e[31m%b\e[0m\n" "$*" +} + +# yellow +warning() { + printf "\e[1;33m%b\e[0m\n" "$*" +} + +# red +debug() { + printf "\e[31m%b\e[0m\n" "[DEBUG] $*" +} + +get_os_version() { + if [ -f /etc/centos-release ]; then + # Older Red Hat, CentOS, Alibaba Cloud Linux etc. + PLATFORM=CentOS + OS_VERSION=$(sed 's/.* \([0-9]\).*/\1/' < /etc/centos-release) + if grep -q "Alibaba Cloud Linux" /etc/centos-release; then + PLATFORM="Aliyun_based_on_CentOS" + OS_VERSION=$(rpm -E %{rhel}) + fi + elif [ -f /etc/os-release ]; then + # freedesktop.org and systemd + . /etc/os-release + PLATFORM="${NAME}" + OS_VERSION="${VERSION_ID}" + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + PLATFORM=$(lsb_release -si) + OS_VERSION=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + PLATFORM="${DISTRIB_ID}" + OS_VERSION="${DISTRIB_RELEASE}" + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + PLATFORM=Debian + OS_VERSION=$(cat /etc/debian_version) + else + # Fall back to uname, e.g. "Linux ", also works for BSD, Darwin, etc. + PLATFORM=$(uname -s) + OS_VERSION=$(uname -r) + fi + if [[ "${PLATFORM}" != *"Ubuntu"* && "${PLATFORM}" != *"CentOS"* && "${PLATFORM}" != *"Darwin"* && "${PLATFORM}" != *"Aliyun"* ]];then + err "Only suppport on Ubuntu/CentOS/macOS/AliyunOS platform." + exit 1 + fi + if [[ "${PLATFORM}" == *"Ubuntu"* && "${OS_VERSION:0:2}" -lt "20" ]]; then + err "Ubuntu ${OS_VERSION} found, requires 20 or greater." + exit 1 + fi + if [[ "${PLATFORM}" == *"CentOS"* && "${OS_VERSION}" -lt "7" ]]; then + err "CentOS ${OS_VERSION} found, requires 8 or greater." + exit 1 + fi + if [[ "${PLATFORM}" == *"Darwin"* ]]; then + export HOMEBREW_NO_INSTALL_CLEANUP=1 + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + fi + echo "$PLATFORM-$OS_VERSION" +} + +# default values +readonly OS=$(get_os_version) +readonly OS_PLATFORM=${OS%-*} +readonly OS_VERSION=${OS#*-} +readonly ARCH=$(uname -m) +readonly OUTPUT_ENV_FILE="${HOME}/.graphscope_env" +if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + readonly HOMEBREW_PREFIX=$(brew --prefix) +fi +readonly ARROW_VERSION="15.0.2" +readonly MPI_PREFIX="/opt/openmpi" # fixed, related to coordinator/setup.py +readonly V6D_PREFIX="/opt/vineyard" # fixed, related to coordinator/setup.py +readonly tempdir="/tmp/gs-local-deps" +v6d_version="main" +no_v6d=false +for_analytical=false +for_analytical_java=false +for_interactive=false +for_learning=false +cn_flag=false +debug_flag=false +install_prefix="/opt/graphscope" + +# parse args +while (( "$#" )); do + case "$1" in + dev) + for_analytical=true + for_interactive=true + for_learning=true + shift + ;; + dev-analytical) + for_analytical=true + shift + ;; + dev-analytical-java) + for_analytical=true + for_analytical_java=true + shift + ;; + dev-interactive) + for_interactive=true + shift + ;; + dev-learning) + for_learning=true + shift + ;; + --install-prefix) + install_prefix="$2" + shift 2 + ;; + --v6d-version) + v6d_version="$2" + shift 2 + ;; + --no-v6d) + no_v6d=true + shift + ;; + --cn) + cn_flag=true + shift + ;; + --debug) + debug_flag=true + shift + ;; + *) + shift + ;; + esac +done + +if [[ ${for_analytical} == false && ${for_interactive} == false && ${for_learning} == false ]]; then + usage="Usage: ${0} dev/dev-analytical/dev-analytical-java/dev-interactive/dev-learning" + usage="${usage} [--cn] [--v6d-version ] [--install-prefix ] [--no-v6d] [--debug]" + err "${usage}" + exit 0 +fi + +if [[ ${debug_flag} == true ]]; then + debug "OS: ${OS}, OS_PLATFORM: ${OS_PLATFORM}, OS_VERSION: ${OS_VERSION}" + debug "install dependencies for analytical=${for_analytical}, analytical-java=${for_analytical_java}, v6d_version=${v6d_version}, no_v6d=${no_v6d}" + debug "install dependencies for interactive=${for_interactive}, learning=${for_learning}" + debug "install prefix: ${install_prefix}" +fi + +# sudo +SUDO=sudo +if [[ $(id -u) -eq 0 ]]; then + SUDO="" +fi + +# speed up +if [ "${cn_flag}" == true ]; then + export HOMEBREW_BREW_GIT_REMOTE="https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/brew.git" + export HOMEBREW_CORE_GIT_REMOTE="https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/homebrew-core.git" + export HOMEBREW_BOTTLE_DOMAIN="https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles" +fi + +# install functions +init_workspace_and_env() { + info "creating directory: ${install_prefix} ${MPI_PREFIX} ${tempdir} ${V6D_PREFIX}" + ${SUDO} mkdir -p ${install_prefix} ${MPI_PREFIX} ${tempdir} ${V6D_PREFIX} + ${SUDO} chown -R $(id -u):$(id -g) ${install_prefix} ${MPI_PREFIX} ${tempdir} ${V6D_PREFIX} + export PATH=${install_prefix}/bin:${PATH} + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 +} + +# utils functions +function set_to_cn_url() { + local url=$1 + if [[ ${cn_flag} == true ]]; then + url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" + fi + echo ${url} +} + +function fetch_source() { + local url=$1 + local file=$2 + curl -fsSL -o "${file}" "${url}/${file}" +} + +function download_and_untar() { + local url=$1 + local file=$2 + local directory=$3 + if [ ! -d "${directory}" ]; then + [ ! -f "${file}" ] && fetch_source "${url}" "${file}" + tar zxf "${file}" + fi +} + +function git_clone() { + local url=$1 + local file=$2 + local directory=$3 + local branch=$4 + if [ ! -d "${directory}" ]; then + if [ ! -f "${file}" ]; then + git clone --depth=1 --branch "${branch}" "${url}" "${directory}" + pushd "${directory}" || exit + git submodule update --init || true + popd || exit + else + tar zxf "${file}" + fi + fi +} + +# cmake for centos +install_cmake() { + if [[ -f "${install_prefix}/bin/cmake" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + file="cmake-3.24.3-linux-${ARCH}.sh" + url="https://github.com/Kitware/CMake/releases/download/v3.24.3" + url=$(set_to_cn_url ${url}) + [ ! -f "${file}" ] && fetch_source "${url}" "${file}" + bash "${file}" --prefix="${install_prefix}" --skip-license + popd || exit + rm -rf "${tempdir:?}/${file:?}" +} + +# gflags for centos +install_gflags() { + if [[ -f "${install_prefix}/include/gflags/gflags.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="gflags-2.2.2" + file="v2.2.2.tar.gz" + url="https://github.com/gflags/gflags/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DBUILD_SHARED_LIBS=ON + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# glog for centos +install_glog() { + if [[ -f "${install_prefix}/include/glog/logging.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="glog-0.6.0" + file="v0.6.0.tar.gz" + url="https://github.com/google/glog/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DBUILD_SHARED_LIBS=ON + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# boost for centos +install_boost() { + if [[ -f "${install_prefix}/include/boost/version.hpp" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="boost_1_74_0" + file="${directory}.tar.gz" + url="https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./bootstrap.sh --prefix="${install_prefix}" \ + --with-libraries=system,filesystem,context,program_options,regex,thread,random,chrono,atomic,date_time + ./b2 install link=shared runtime-link=shared variant=release threading=multi + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# openssl for centos +install_openssl() { + if [[ -f "${install_prefix}/include/openssl/ssl.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="openssl-OpenSSL_1_1_1k" + file="OpenSSL_1_1_1k.tar.gz" + url="https://github.com/openssl/openssl/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./config --prefix="${install_prefix}" -fPIC -shared + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# static openssl for centos 8 +install_openssl_static() { + if [[ -f "${install_prefix}/include/openssl/ssl.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="openssl-OpenSSL_1_1_1k" + file="OpenSSL_1_1_1k.tar.gz" + url="https://github.com/openssl/openssl/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./config --prefix="${install_prefix}" no-shared -fPIC + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# arrow for centos +install_apache_arrow() { + if [[ -f "${install_prefix}/include/arrow/api.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="arrow-apache-arrow-${ARROW_VERSION}" + file="apache-arrow-${ARROW_VERSION}.tar.gz" + url="https://github.com/apache/arrow/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + cmake ./cpp \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DARROW_COMPUTE=ON \ + -DARROW_WITH_UTF8PROC=OFF \ + -DARROW_CSV=ON \ + -DARROW_CUDA=OFF \ + -DARROW_DATASET=OFF \ + -DARROW_FILESYSTEM=ON \ + -DARROW_FLIGHT=OFF \ + -DARROW_GANDIVA=OFF \ + -DARROW_HDFS=OFF \ + -DARROW_JSON=OFF \ + -DARROW_ORC=OFF \ + -DARROW_PARQUET=OFF \ + -DARROW_PLASMA=OFF \ + -DARROW_PYTHON=OFF \ + -DARROW_S3=OFF \ + -DARROW_WITH_BZ2=OFF \ + -DARROW_WITH_ZLIB=OFF \ + -DARROW_WITH_LZ4=OFF \ + -DARROW_WITH_SNAPPY=OFF \ + -DARROW_WITH_ZSTD=OFF \ + -DARROW_WITH_BROTLI=OFF \ + -DARROW_IPC=ON \ + -DARROW_BUILD_BENCHMARKS=OFF \ + -DARROW_BUILD_EXAMPLES=OFF \ + -DARROW_BUILD_INTEGRATION=OFF \ + -DARROW_BUILD_UTILITIES=OFF \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_ENABLE_TIMING_TESTS=OFF \ + -DARROW_FUZZING=OFF \ + -DARROW_USE_ASAN=OFF \ + -DARROW_USE_TSAN=OFF \ + -DARROW_USE_UBSAN=OFF \ + -DARROW_JEMALLOC=OFF \ + -DARROW_BUILD_SHARED=ON \ + -DARROW_BUILD_STATIC=OFF + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# openmpi for centos +install_openmpi() { + if [[ -f "${install_prefix}/include/mpi.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="openmpi-4.0.5" + file="openmpi-4.0.5.tar.gz" + url="https://download.open-mpi.org/release/open-mpi/v4.0" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./configure --enable-mpi-cxx --disable-dlopen --prefix=${MPI_PREFIX} + make -j$(nproc) + make install + popd || exit + popd || exit + cp -rs ${MPI_PREFIX}/* "${install_prefix}" + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# protobuf for centos +install_protobuf() { + if [[ -f "${install_prefix}/include/google/protobuf/port.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="protobuf-21.9" + file="protobuf-all-21.9.tar.gz" + url="https://github.com/protocolbuffers/protobuf/releases/download/v21.9" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./configure --prefix="${install_prefix}" --enable-shared --disable-static + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# zlib for centos +install_zlib() { + if [[ -f "${install_prefix}/include/zlib.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="zlib-1.2.11" + file="v1.2.11.tar.gz" + url="https://github.com/madler/zlib/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DBUILD_SHARED_LIBS=ON + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# grpc for centos +install_grpc() { + if [[ -f "${install_prefix}/include/grpcpp/grpcpp.h" ]]; then + return 0 + fi + directory="grpc" + branch="v1.49.1" + file="${directory}-${branch}.tar.gz" + url="https://github.com/grpc/grpc.git" + url=$(set_to_cn_url ${url}) + pushd "${tempdir}" || exit + if [[ ${url} == *.git ]]; then + git_clone "${url}" "${file}" "${directory}" "${branch}" + else + download_and_untar "${url}" "${file}" "${directory}" + fi + pushd ${directory} || exit + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DBUILD_SHARED_LIBS=ON \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ + -DgRPC_BACKWARDS_COMPATIBILITY_MODE=ON \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DOPENSSL_ROOT_DIR="${install_prefix}" \ + -DCMAKE_CXX_FLAGS="-fpermissive" \ + -DPNG_ARM_NEON_OPT=0 + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# patchelf +install_patchelf() { + if [[ -f "${install_prefix}/bin/patchelf" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="patchelf" # patchelf doesn't have a folder + file="patchelf-0.14.5-${ARCH}.tar.gz" + url="https://github.com/NixOS/patchelf/releases/download/0.14.5" + url=$(set_to_cn_url ${url}) + mkdir -p "${directory}" + pushd "${directory}" || exit + download_and_untar "${url}" "${file}" "${directory}" + mkdir -p ${install_prefix}/bin + mv bin/patchelf ${install_prefix}/bin/patchelf + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# vineyard +install_vineyard() { + if [[ -f "${V6D_PREFIX}/include/vineyard/client/client.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + python3 -m pip --no-cache-dir install pip -U --user + python3 -m pip --no-cache-dir install libclang wheel auditwheel --user + auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')") + sed -i 's/p.error/logger.warning/g' ${auditwheel_path} + if [[ "${v6d_version}" != "v"* ]]; then + directory="v6d" + file="${directory}-${v6d_version}.tar.gz" + url="https://github.com/v6d-io/v6d.git" + git_clone "${url}" "${file}" "${directory}" "${v6d_version}" + else + # remove the prefix 'v' + directory="v6d-${v6d_version:1:100}" + file="${directory}.tar.gz" + url="https://github.com/v6d-io/v6d/releases/download/${v6d_version}" + download_and_untar "${url}" "${file}" "${directory}" + fi + pushd ${directory} || exit + + # make sure it complain loudly if installing vineyard fails + set -e + cmake . -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DCMAKE_INSTALL_PREFIX="${V6D_PREFIX}" \ + -DBUILD_VINEYARD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \ + -DBUILD_VINEYARD_GRAPH_WITH_GAR=ON + make -j$(nproc) + make install + strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard* + pip3 install --no-cache -i https://pypi.org/simple -U "vineyard" "vineyard-io" + cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/ + set +e + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# java +install_java_and_maven() { + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + if ! command -v javac &>/dev/null; then + brew install --ignore-dependencies openjdk@11 + fi + if ! command -v mvn &>/dev/null; then + brew install --ignore-dependencies maven + fi + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + if ! command -v javac &>/dev/null; then + ${SUDO} apt-get install default-jdk -y + fi + if ! command -v mvn &>/dev/null; then + ${SUDO} apt-get install maven -y + fi + else + if ! command -v javac &>/dev/null; then + ${SUDO} yum install java-11-openjdk-devel -y + fi + if ! command -v mvn &>/dev/null; then + install_maven + fi + fi +} + +# maven for centos +install_maven() { + if [[ -f "${install_prefix}/bin/mvn" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="apache-maven-3.8.6" + file="apache-maven-3.8.6-bin.tar.gz" + url="https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + cp -r ${directory} "${install_prefix}"/ + mkdir -p "${install_prefix}"/bin + ln -s "${install_prefix}/${directory}/bin/mvn" "${install_prefix}/bin/mvn" + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +install_hiactor() { + if [[ -f "${install_prefix}/include/hiactor/core/actor_core.hh" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch + cd hiactor && git submodule update --init --recursive + sudo bash ./seastar/seastar/install-dependencies.sh + mkdir build && cd build + cmake -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DHiactor_CXX_DIALECT=gnu++17 -DSeastar_CXX_FLAGS="-DSEASTAR_DEFAULT_ALLOCATOR -mno-avx512" .. + make -j$(nproc) && make install + popd && rm -rf "${tempdir}/hiactor" +} + + +install_cppkafka() { + if [[ -f "${install_prefix}/include/cppkafka/cppkafka.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="cppkafka-0.4.0" + file="0.4.0.tar.gz" + url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + # cppkafka may not find the lib64 directory + export LIBRARY_PATH=${LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DCPPKAFKA_DISABLE_TESTS=ON \ + -DCPPKAFKA_DISABLE_EXAMPLES=ON + make -j4 + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + + +BASIC_PACKAGES_LINUX=("file" "curl" "wget" "git" "sudo") +BASIC_PACKAGES_UBUNTU=("${BASIC_PACKAGES_LINUX[@]}" "build-essential" "cmake" "libunwind-dev" "python3-pip") +BASIC_PACKAGES_CENTOS_8=("${BASIC_PACKAGES_LINUX[@]}" "epel-release" "libunwind-devel" "libcurl-devel" "perl" "which") +BASIC_PACKAGES_CENTOS_7=("${BASIC_PACKAGES_CENTOS_8[@]}" "centos-release-scl-rh") +ADDITIONAL_PACKAGES_CENTOS_8=("gcc-c++" "python38-devel") +ADDITIONAL_PACKAGES_CENTOS_7=("make" "devtoolset-8-gcc-c++" "rh-python38-python-pip" "rh-python38-python-devel") + +install_basic_packages() { + if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + ${SUDO} apt-get update -y + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${BASIC_PACKAGES_UBUNTU[*]} + elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then + ${SUDO} yum update -y + if [[ "${OS_VERSION}" -eq "7" ]]; then + # centos7 + ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_7[*]} + # change the source for centos-release-scl-rh + ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} sed -i 's|# baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_7[*]} + else + if [[ "${OS_PLATFORM}" == *"Aliyun"* ]]; then + ${SUDO} yum install -y 'dnf-command(config-manager)' + ${SUDO} dnf install -y epel-release --allowerasing + else + ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + ${SUDO} yum install -y 'dnf-command(config-manager)' + ${SUDO} dnf install -y epel-release + ${SUDO} dnf config-manager --set-enabled powertools + fi + ${SUDO} dnf config-manager --set-enabled epel + ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_8[*]} + ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_8[*]} + fi + fi +} + +ANALYTICAL_MACOS=("apache-arrow" "boost" "gflags" "glog" "open-mpi" "openssl@1.1" "protobuf" "grpc" "rapidjson" "msgpack-cxx" "librdkafka" "patchelf") +ANALYTICAL_UBUNTU=("libboost-all-dev" "libopenmpi-dev" "libgflags-dev" "libgoogle-glog-dev" "libprotobuf-dev" "libgrpc++-dev" "libmsgpack-dev" "librdkafka-dev" "protobuf-compiler-grpc" "rapidjson-dev") +ANALYTICAL_CENTOS_7=("librdkafka-devel" "msgpack-devel" "rapidjson-devel") +ANALYTICAL_CENTOS_8=("${ANALYTICAL_CENTOS_7[@]}" "boost-devel" "gflags-devel" "glog-devel") + +install_analytical_centos_common_dependencies() { + install_patchelf + install_apache_arrow + install_openmpi + install_protobuf + install_zlib + install_grpc +} + +install_analytical_dependencies() { + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + brew install ${ANALYTICAL_MACOS[*]} + export LDFLAGS="${LDFLAGS} -L${HOMEBREW_PREFIX}/opt/openssl@1.1/lib" + export CPPFLAGS="${CPPFLAGS} -I${HOMEBREW_PREFIX}/opt/openssl@1.1/include" + export PKG_CONFIG_PATH="${HOMEBREW_PREFIX}/opt/openssl@1.1/lib/pkgconfig" + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${ANALYTICAL_UBUNTU[*]} + # patchelf + install_patchelf + # arrow + if ! dpkg -s libarrow-dev &>/dev/null; then + ${SUDO} apt-get install -y lsb-release + # shellcheck disable=SC2046,SC2019,SC2018 + wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -P /tmp/ + ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb + ${SUDO} apt-get update -y + ${SUDO} apt-get install -y libarrow-dev=${ARROW_VERSION}-1 libarrow-dataset-dev=${ARROW_VERSION}-1 libarrow-acero-dev=${ARROW_VERSION}-1 libparquet-dev=${ARROW_VERSION}-1 + rm /tmp/apache-arrow-apt-source-latest-*.deb + fi + else + if [[ "${OS_VERSION}" -eq "7" ]]; then + ${SUDO} yum install -y ${ANALYTICAL_CENTOS_7[*]} + source /opt/rh/devtoolset-8/enable + source /opt/rh/rh-python38/enable + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 + install_cmake + install_gflags + install_glog + install_boost + # must to install openssl before grpc + install_openssl + else + ${SUDO} yum install -y ${ANALYTICAL_CENTOS_8[*]} + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib/:/lib64${install_prefix}/lib:${install_prefix}/lib64 + install_cmake + install_openssl_static + fi + install_analytical_centos_common_dependencies + fi + # install java for gae-java + install_java_and_maven + # install vineyard + if [[ "${no_v6d}" != true ]]; then + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + brew install vineyard + else + install_vineyard + fi + fi +} + +install_analytical_java_dependencies() { + # llvm + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + brew install llvm || true # prevent the `brew link` failure + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + ${SUDO} apt-get install -y llvm-11-dev lld-11 clang-11 + else + if [[ "${OS_VERSION}" -eq "7" ]]; then + ${SUDO} yum install -y llvm-toolset-7.0-clang-devel + source /opt/rh/llvm-toolset-7.0/enable + else + ${SUDO} yum install -y llvm-devel clang-devel lld + fi + fi +} + +install_interactive_dependencies() { + # java + install_java_and_maven + # rust + if ! command -v rustup &>/dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source $HOME/.cargo/env + rustup install 1.71.0 + rustup default 1.71.0 + rustc --version + fi + # hiactor + if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + install_hiactor + else + warning "Skip installing dependencies for flex interactive on ${OS_PLATFORM}." + fi +} + +install_learning_dependencies() { + install_cppkafka +} + +write_env_config() { + echo "" > ${OUTPUT_ENV_FILE} + # common + { + echo "export GRAPHSCOPE_HOME=${install_prefix}" + echo "export CMAKE_PREFIX_PATH=/opt/vineyard:/opt/graphscope/" + echo "export PATH=${install_prefix}/bin:\$HOME/.local/bin:\$HOME/.cargo/bin:\$PATH" + echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" + echo "export LIBRARY_PATH=${install_prefix}/lib:${install_prefix}/lib64" + } >> "${OUTPUT_ENV_FILE}" + { + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl" + echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib" + echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib" + elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then + if [[ "${OS_VERSION}" -eq "7" ]]; then + echo "source /opt/rh/devtoolset-8/enable" + echo "source /opt/rh/rh-python38/enable" + fi + echo "export OPENSSL_ROOT_DIR=${install_prefix}" + fi + } >> "${OUTPUT_ENV_FILE}" + # JAVA_HOME + { + if [[ "${for_analytical}" == true || "${for_interactive}" == true ]]; then + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + if [ -z "${JAVA_HOME}" ]; then + echo "export JAVA_HOME=\$(/usr/libexec/java_home -v11)" + fi + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + if [ -z "${JAVA_HOME}" ]; then + echo "export JAVA_HOME=/usr/lib/jvm/default-java" + fi + else + if [ -z "${JAVA_HOME}" ]; then + echo "export OPENSSL_ROOT_DIR=${install_prefix}" + fi + fi + fi + } >> "${OUTPUT_ENV_FILE}" + { + if [[ "${for_analytical_java}" == true ]]; then + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + echo "export CC=${homebrew_prefix}/opt/llvm/bin/clang" + echo "export CXX=${homebrew_prefix}/opt/llvm/bin/clang++" + echo "export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}" + echo "export LDFLAGS=\"-L${homebrew_prefix}/opt/llvm/lib\"" + echo "export CPPFLAGS=\"-I${homebrew_prefix}/opt/llvm/include\"" + elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then + echo "source /opt/rh/llvm-toolset-7.0/enable || true" + echo "export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/" + fi + fi + } >> "${OUTPUT_ENV_FILE}" +} + +install_deps() { + init_workspace_and_env + install_basic_packages + [[ "${for_analytical}" == true ]] && install_analytical_dependencies + [[ "${for_analytical_java}" == true ]] && install_analytical_java_dependencies + [[ "${for_interactive}" == true ]] && install_interactive_dependencies + [[ "${for_learning}" == true ]] && install_learning_dependencies + write_env_config + info "The script has installed all dependencies, don't forget to exec command:\n + $ source ${OUTPUT_ENV_FILE} + \nbefore building GraphScope." +} + +install_deps diff --git a/python/graphscope/gsctl/scripts/install_deps_command.sh b/python/graphscope/gsctl/scripts/install_deps_command.sh deleted file mode 100644 index a835b690f8cd..000000000000 --- a/python/graphscope/gsctl/scripts/install_deps_command.sh +++ /dev/null @@ -1,448 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh -source ${script_dir}/initialize.sh - -# parse args -while getopts ":t:i:d:v:j:-:" opt; do - case ${opt} in - t) - type=${OPTARG} - ;; - i) - install_prefix=${OPTARG} - ;; - d) - deps_prefix=${OPTARG} - ;; - v) - v6d_version=${OPTARG} - ;; - j) - jobs=${OPTARG} - ;; - -) - case ${OPTARG} in - for-analytical) - for_analytical=true - ;; - no-v6d) - no_v6d=true - ;; - cn) - cn=true - ;; - *) - echo "Invalid option: --${OPTARG}" - exit 1 - ;; - esac - ;; - *) - echo "Invalid option: -${OPTARG}" - exit 1 - ;; - esac -done -shift $((OPTIND -1)) - -SUDO=sudo -if [[ $(id -u) -eq 0 ]]; then - warning "Please note that I am running as root." - SUDO="" -fi - -readonly OS=$(get_os_version) -readonly OS_PLATFORM=${OS%-*} -readonly OS_VERSION=${OS#*-} - -readonly OUTPUT_ENV_FILE="${HOME}/.graphscope_env" - -log "Installing ${type} dependencies for GraphScope on ${OS}..." - -if [ "${cn}" == true ]; then - log "Set to speed up downloading for CN locations." - # export some mirror locations for CN, e.g., brew/docker... - export HOMEBREW_BREW_GIT_REMOTE="https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/brew.git" - export HOMEBREW_CORE_GIT_REMOTE="https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/homebrew-core.git" - export HOMEBREW_BOTTLE_DOMAIN="https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles" - export GRAPHSCOPE_DOWNLOAD_FROM_CN="true" -fi - -check_os_compatibility() { - if [[ "${OS_PLATFORM}" != *"Ubuntu"* && "${OS_PLATFORM}" != *"CentOS"* && "${OS_PLATFORM}" != *"Darwin"* && "${OS_PLATFORM}" != *"Aliyun"* ]]; then - err "The script is only support platforms of Ubuntu/CentOS/macOS/AliyunOS" - exit 1 - fi - - if [[ "${OS_PLATFORM}" == *"Ubuntu"* && "${OS_VERSION:0:2}" -lt "20" ]]; then - err "The version of Ubuntu is ${OS_VERSION}. This script requires Ubuntu 20 or greater." - exit 1 - fi - - if [[ "${OS_PLATFORM}" == *"CentOS"* && "${OS_VERSION}" -lt "7" ]]; then - err "The version of CentOS is ${OS_VERSION}. This script requires CentOS 8 or greater." - exit 1 - fi - - log "Running on ${OS_PLATFORM} ${OS_VERSION}" -} - -BASIC_PACKAGES_LINUX=("file" "curl" "wget" "git" "sudo") - -BASIC_PACKAGES_UBUNTU=("${BASIC_PACKAGES_LINUX[@]}" "build-essential" "cmake" "libunwind-dev" "python3-pip") - -BASIC_PACKAGES_CENTOS_8=("${BASIC_PACKAGES_LINUX[@]}" "epel-release" "libunwind-devel" "libcurl-devel" "perl" "which") -BASIC_PACKAGES_CENTOS_7=("${BASIC_PACKAGES_CENTOS_8[@]}" "centos-release-scl-rh") -ADDITIONAL_PACKAGES_CENTOS_8=("gcc-c++" "python38-devel") -ADDITIONAL_PACKAGES_CENTOS_7=("make" "devtoolset-8-gcc-c++" "rh-python38-python-pip" "rh-python38-python-devel") - -ANALYTICAL_UBUNTU=( - "libboost-all-dev" - "libopenmpi-dev" - "libgflags-dev" - "libgoogle-glog-dev" - "libprotobuf-dev" - "libgrpc++-dev" - "libmsgpack-dev" - "librdkafka-dev" - "protobuf-compiler-grpc" - "rapidjson-dev" -) - -ANALYTICAL_CENTOS_7=("librdkafka-devel" "msgpack-devel" "rapidjson-devel") -ANALYTICAL_CENTOS_8=("${ANALYTICAL_CENTOS_7[@]}" "boost-devel" "gflags-devel" "glog-devel") - -ANALYTICAL_MACOS=( - "apache-arrow" - "boost" - "gflags" - "glog" - "open-mpi" - "openssl@1.1" - "protobuf" - "grpc" - "rapidjson" - "msgpack-cxx" - "librdkafka" - "patchelf" -) - -_install_apache_arrow_ubuntu() { - if ! dpkg -s libarrow-dev &>/dev/null; then - log "Installing apache-arrow." - ${SUDO} apt-get install -y lsb-release - # shellcheck disable=SC2046,SC2019,SC2018 - wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ - -P /tmp/ - ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb - ${SUDO} apt-get update -y - ${SUDO} apt-get install -y libarrow-dev=15.0.2-1 libarrow-dataset-dev=15.0.2-1 libarrow-acero-dev=15.0.2-1 libparquet-dev=15.0.2-1 - #TODO(zhanglei): Remove the arrow version constrait after new version of vineyard is release. see https://github.com/v6d-io/v6d/pull/1911 and https://github.com/apache/incubator-graphar/issues/515 - #${SUDO} apt-get install -y libarrow-dev libarrow-dataset-dev libarrow-acero-dev libparquet-dev - rm /tmp/apache-arrow-apt-source-latest-*.deb - else - log "apache-arrow (libarrow-dev) already installed, skip." - fi -} - -_install_java_maven_ubuntu() { - if ! command -v javac &>/dev/null; then - log "Installing default-jdk" - ${SUDO} apt-get install default-jdk -y - fi - if ! command -v mvn &>/dev/null; then - log "Installing maven" - ${SUDO} apt-get install maven -y - fi -} - -_install_java_maven_centos() { - if ! command -v javac &>/dev/null; then - log "Installing java-11-openjdk-devel" - ${SUDO} yum install java-11-openjdk-devel -y - fi - if ! command -v mvn &>/dev/null; then - log "Installing maven" - install_maven "${deps_prefix}" "${install_prefix}" - fi -} - -_install_java_maven_macos() { - if ! command -v javac &>/dev/null; then - log "Installing openjdk@11" - # we need arm64-base jvm, install from brew. - brew install --ignore-dependencies openjdk@11 - fi - if ! command -v mvn &>/dev/null; then - log "Installing maven" - brew install --ignore-dependencies maven - fi -} - -_install_dependencies_analytical_ubuntu() { - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${ANALYTICAL_UBUNTU[*]} - _install_apache_arrow_ubuntu -} - -_install_dependencies_analytical_centos_common() { - # the openssl must put before grpc, otherwise the grpc - # cannot find the openssl. - install_apache_arrow "${deps_prefix}" "${install_prefix}" - install_open_mpi "${deps_prefix}" "${install_prefix}" - install_protobuf "${deps_prefix}" "${install_prefix}" - install_zlib "${deps_prefix}" "${install_prefix}" - install_grpc "${deps_prefix}" "${install_prefix}" -} - -_install_dependencies_analytical_centos8() { - ${SUDO} yum install -y ${ANALYTICAL_CENTOS_8[*]} - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib/:/lib64${install_prefix}/lib:${install_prefix}/lib64 - install_cmake "${deps_prefix}" "${install_prefix}" - install_openssl_static "${deps_prefix}" "${install_prefix}" - _install_dependencies_analytical_centos_common -} -_install_dependencies_analytical_centos7() { - ${SUDO} yum install -y ${ANALYTICAL_CENTOS_7[*]} - source /opt/rh/devtoolset-8/enable - source /opt/rh/rh-python38/enable - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 - - install_cmake "${deps_prefix}" "${install_prefix}" - install_gflags "${deps_prefix}" "${install_prefix}" - install_glog "${deps_prefix}" "${install_prefix}" - install_boost "${deps_prefix}" "${install_prefix}" - install_openssl "${deps_prefix}" "${install_prefix}" - _install_dependencies_analytical_centos_common -} -_install_dependencies_analytical_macos() { - brew install ${ANALYTICAL_MACOS[*]} - homebrew_prefix=$(brew --prefix) - export LDFLAGS="${LDFLAGS} -L${homebrew_prefix}/opt/openssl@1.1/lib" - export CPPFLAGS="${CPPFLAGS} -I${homebrew_prefix}/opt/openssl@1.1/include" - export PKG_CONFIG_PATH="${homebrew_prefix}/opt/openssl@1.1/lib/pkgconfig" -} - -install_basic_packages_universal() { - if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - ${SUDO} apt-get update -y - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${BASIC_PACKAGES_UBUNTU[*]} - elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then - if [[ "${OS_VERSION}" -eq "7" ]]; then - ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_7[*]} - # change the source for centos-release-scl-rh - ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*scl* - ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* - ${SUDO} sed -i 's|# baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* - ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_7[*]} - else - if [[ "${OS_PLATFORM}" == *"Aliyun"* ]]; then - ${SUDO} yum install -y 'dnf-command(config-manager)' - ${SUDO} dnf install -y epel-release --allowerasing - else - ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - ${SUDO} yum install -y 'dnf-command(config-manager)' - ${SUDO} dnf install -y epel-release - ${SUDO} dnf config-manager --set-enabled powertools - fi - ${SUDO} dnf config-manager --set-enabled epel - ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_8[*]} - ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_8[*]} - fi - fi -} - -install_grape_vineyard_universal() { - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - brew install vineyard - else - log "Installing python packages for vineyard codegen." - pip3 --no-cache-dir install pip -U --user - pip3 --no-cache-dir install libclang wheel auditwheel --user - install_vineyard "${deps_prefix}" "${install_prefix}" "${v6d_version}" "${jobs}" - fi -} - -install_rust_universal() { - if ! command -v rustup &>/dev/null; then - log "Installing rust with fixed version: 1.71/0." - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source $HOME/.cargo/env - rustup install 1.71.0 - rustup default 1.71.0 - rustc --version - fi -} - -install_java_maven_universal() { - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - _install_java_maven_macos - elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - _install_java_maven_ubuntu - else - _install_java_maven_centos - fi -} - -install_llvm_universal() { - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - brew install llvm || true # prevent the `brew link` failure - homebrew_prefix=$(brew --prefix) - export CC=${homebrew_prefix}/opt/llvm/bin/clang - export CXX=${homebrew_prefix}/opt/llvm/bin/clang++ - export CPPFLAGS="${CPPFLAGS} -I${homebrew_prefix}/opt/llvm/include" - export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC} - elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - ${SUDO} apt-get install -y llvm-11-dev lld-11 clang-11 - else - if [[ "${OS_VERSION}" -eq "7" ]]; then - ${SUDO} yum install -y llvm-toolset-7.0-clang-devel - source /opt/rh/llvm-toolset-7.0/enable - export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/ - else - ${SUDO} yum install -y llvm-devel clang-devel lld - fi - fi -} - -install_dependencies_analytical_universal() { - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - _install_dependencies_analytical_macos - elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - install_patchelf "${deps_prefix}" "${install_prefix}" - _install_dependencies_analytical_ubuntu - else - install_patchelf "${deps_prefix}" "${install_prefix}" - if [[ "${OS_VERSION}" -eq "7" ]]; then - _install_dependencies_analytical_centos7 - install_java_maven_universal - else - _install_dependencies_analytical_centos8 - fi - fi -} - -install_interactive_deps() { - # seastar can not be built on macos and centos7 - if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - install_hiactor "${install_prefix}" - else - warning "Skip installing dependencies for flex interactive on ${OS_PLATFORM}." - fi -} - -write_env_config() { - log "Output environments config file ${OUTPUT_ENV_FILE}" - if [ -f "${OUTPUT_ENV_FILE}" ]; then - warning "Found ${OUTPUT_ENV_FILE} exists, remove the environment config file and generate a new one." - rm -f "${OUTPUT_ENV_FILE}" - fi - - { - echo "export GRAPHSCOPE_HOME=${install_prefix}" - echo "export CMAKE_PREFIX_PATH=/opt/vineyard:/opt/graphscope/" - echo "export PATH=${install_prefix}/bin:\$HOME/.local/bin:\$HOME/.cargo/bin:\$PATH" - echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" - echo "export LIBRARY_PATH=${install_prefix}/lib:${install_prefix}/lib64" - } >>"${OUTPUT_ENV_FILE}" - - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - homebrew_prefix=$(brew --prefix) - { - echo "export CC=${homebrew_prefix}/opt/llvm/bin/clang" - echo "export CXX=${homebrew_prefix}/opt/llvm/bin/clang++" - echo "export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}" - if [ -z "${JAVA_HOME}" ]; then - echo "export JAVA_HOME=\$(/usr/libexec/java_home -v11)" - fi - echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl" - echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib" - echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib" - echo "export LDFLAGS=\"-L${homebrew_prefix}/opt/llvm/lib\"" - echo "export CPPFLAGS=\"-I${homebrew_prefix}/opt/llvm/include\"" - } >>"${OUTPUT_ENV_FILE}" - - elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - { - if [ -z "${JAVA_HOME}" ]; then - echo "export JAVA_HOME=/usr/lib/jvm/default-java" - fi - } >>"${OUTPUT_ENV_FILE}" - else - { - if [[ "${OS_VERSION}" -eq "7" ]]; then - echo "source /opt/rh/devtoolset-8/enable" - echo "source /opt/rh/rh-python38/enable" - echo "source /opt/rh/llvm-toolset-7.0/enable || true" - echo "export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/" - fi - if [ -z "${JAVA_HOME}" ]; then - echo "export JAVA_HOME=/usr/lib/jvm/jre-openjdk" - fi - echo "export OPENSSL_ROOT_DIR=${install_prefix}" - } >>"${OUTPUT_ENV_FILE}" - fi -} - -init_workspace_and_env() { - mkdir -p "${install_prefix}" - mkdir -p "${deps_prefix}" - export PATH=${install_prefix}/bin:${PATH} - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 -} - -install_deps_for_dev() { - # install_deps for development on local - check_os_compatibility - - init_workspace_and_env - - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - export HOMEBREW_NO_INSTALL_CLEANUP=1 - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - fi - - install_basic_packages_universal - if [ "${for_analytical}" == true ]; then - install_dependencies_analytical_universal - if [ "${no_v6d}" != true ]; then - install_grape_vineyard_universal - fi - else # for all - install_dependencies_analytical_universal - if [ "${no_v6d}" != true ]; then - install_grape_vineyard_universal - fi - install_java_maven_universal - install_llvm_universal - install_rust_universal - install_cppkafka "${deps_prefix}" "${install_prefix}" - # install dependencies for flex interactive - install_interactive_deps - fi - - write_env_config - - succ "The script has installed all dependencies for building GraphScope, use commands:\n - $ source ${OUTPUT_ENV_FILE} - $ make install - \nto build and develop GraphScope." -} - -install_deps_for_client() { - # install python.. - # TODO: refine - pip3 --no-cache-dir install -U pip --user - pip3 --no-cache-dir install auditwheel daemons etcd-distro gremlinpython \ - hdfs3 fsspec oss2 s3fs ipython kubernetes libclang networkx==2.4 numpy pandas parsec pycryptodome \ - pyorc pytest scipy scikit_learn wheel --user - pip3 --no-cache-dir install Cython --pre -U --user -} - -# run subcommand with the type -install_deps_for_"${type}" diff --git a/python/graphscope/gsctl/scripts/lib/colors.sh b/python/graphscope/gsctl/scripts/lib/colors.sh deleted file mode 100644 index 25bc07e9a3d9..000000000000 --- a/python/graphscope/gsctl/scripts/lib/colors.sh +++ /dev/null @@ -1,42 +0,0 @@ -## Color functions [@bashly-upgrade colors] -## This file is a part of Bashly standard library -## -## Usage: -## Use any of the functions below to color or format a portion of a string. -## -## echo "before $(red this is red) after" -## echo "before $(green_bold this is green_bold) after" -## -## Color output will be disabled if `NO_COLOR` environment variable is set -## in compliance with https://no-color.org/ -## -print_in_color() { - local color="$1" - shift - if [[ -z ${NO_COLOR+x} ]]; then - printf "$color%b\e[0m\n" "$*"; - else - printf "%b\n" "$*"; - fi -} - -red() { print_in_color "\e[31m" "$*"; } -green() { print_in_color "\e[32m" "$*"; } -yellow() { print_in_color "\e[33m" "$*"; } -blue() { print_in_color "\e[34m" "$*"; } -magenta() { print_in_color "\e[35m" "$*"; } -cyan() { print_in_color "\e[36m" "$*"; } -bold() { print_in_color "\e[1m" "$*"; } -underlined() { print_in_color "\e[4m" "$*"; } -red_bold() { print_in_color "\e[1;31m" "$*"; } -green_bold() { print_in_color "\e[1;32m" "$*"; } -yellow_bold() { print_in_color "\e[1;33m" "$*"; } -blue_bold() { print_in_color "\e[1;34m" "$*"; } -magenta_bold() { print_in_color "\e[1;35m" "$*"; } -cyan_bold() { print_in_color "\e[1;36m" "$*"; } -red_underlined() { print_in_color "\e[4;31m" "$*"; } -green_underlined() { print_in_color "\e[4;32m" "$*"; } -yellow_underlined() { print_in_color "\e[4;33m" "$*"; } -blue_underlined() { print_in_color "\e[4;34m" "$*"; } -magenta_underlined() { print_in_color "\e[4;35m" "$*"; } -cyan_underlined() { print_in_color "\e[4;36m" "$*"; } diff --git a/python/graphscope/gsctl/scripts/lib/get_os_version.sh b/python/graphscope/gsctl/scripts/lib/get_os_version.sh deleted file mode 100644 index 6a306b5446cf..000000000000 --- a/python/graphscope/gsctl/scripts/lib/get_os_version.sh +++ /dev/null @@ -1,34 +0,0 @@ -get_os_version() { - if [ -f /etc/centos-release ]; then - # Older Red Hat, CentOS, Alibaba Cloud Linux etc. - PLATFORM=CentOS - OS_VERSION=$(sed 's/.* \([0-9]\).*/\1/' < /etc/centos-release) - if grep -q "Alibaba Cloud Linux" /etc/centos-release; then - PLATFORM="Aliyun_based_on_CentOS" - OS_VERSION=$(rpm -E %{rhel}) - fi - elif [ -f /etc/os-release ]; then - # freedesktop.org and systemd - . /etc/os-release - PLATFORM="${NAME}" - OS_VERSION="${VERSION_ID}" - elif type lsb_release >/dev/null 2>&1; then - # linuxbase.org - PLATFORM=$(lsb_release -si) - OS_VERSION=$(lsb_release -sr) - elif [ -f /etc/lsb-release ]; then - # For some versions of Debian/Ubuntu without lsb_release command - . /etc/lsb-release - PLATFORM="${DISTRIB_ID}" - OS_VERSION="${DISTRIB_RELEASE}" - elif [ -f /etc/debian_version ]; then - # Older Debian/Ubuntu/etc. - PLATFORM=Debian - OS_VERSION=$(cat /etc/debian_version) - else - # Fall back to uname, e.g. "Linux ", also works for BSD, Darwin, etc. - PLATFORM=$(uname -s) - OS_VERSION=$(uname -r) - fi - echo "$PLATFORM-$OS_VERSION" -} diff --git a/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh b/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh deleted file mode 100644 index 704b61ec6d74..000000000000 --- a/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh +++ /dev/null @@ -1,472 +0,0 @@ -install_cmake() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/bin/cmake" ]]; then - log "cmake already installed, skip." - return 0 - fi - - ARCH=$(uname -m) - file="cmake-3.24.3-linux-${ARCH}.sh" - log "Building and installing ${file}." - pushd "${workdir}" || exit - - url="https://github.com/Kitware/CMake/releases/download/v3.24.3" - url=$(maybe_set_to_cn_url ${url}) - [ ! -f "${file}" ] && - fetch_source "${file}" "${url}" - bash "${file}" --prefix="${install_prefix}" --skip-license - popd || exit - cleanup_files "${workdir}/${file}" -} - -install_open_mpi() { - workdir=$1 - install_prefix=$2 - MPI_PREFIX="/opt/openmpi" # fixed, related to coordinator/setup.py - - if [[ -f "${install_prefix}/include/mpi.h" ]]; then - log "openmpi already installed, skip." - return 0 - fi - - directory="openmpi-4.0.5" - file="openmpi-4.0.5.tar.gz" - url="https://download.open-mpi.org/release/open-mpi/v4.0" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - ./configure --enable-mpi-cxx --disable-dlopen --prefix=${MPI_PREFIX} - make -j$(nproc) - make install - popd || exit - popd || exit - cp -rs ${MPI_PREFIX}/* "${install_prefix}" - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_gflags() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/gflags/gflags.h" ]]; then - log "gflags already installed, skip." - return 0 - fi - - directory="gflags-2.2.2" - file="v2.2.2.tar.gz" - url="https://github.com/gflags/gflags/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing gflags-2.2.2." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DBUILD_SHARED_LIBS=ON - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_glog() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/glog/logging.h" ]]; then - log "glog already installed, skip." - return 0 - fi - - directory="glog-0.6.0" - file="v0.6.0.tar.gz" - url="https://github.com/google/glog/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DBUILD_SHARED_LIBS=ON - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_apache_arrow() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/arrow/api.h" ]]; then - log "apache-arrow already installed, skip." - return 0 - fi - - directory="arrow-apache-arrow-10.0.1" - file="apache-arrow-10.0.1.tar.gz" - url="https://github.com/apache/arrow/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - # temporarily fix the xsimd dependency downloading issue by manually download - mkdir -p src && wget https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies/9.0.1.tar.gz -P src/ - - cmake ./cpp \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DARROW_COMPUTE=ON \ - -DARROW_WITH_UTF8PROC=OFF \ - -DARROW_CSV=ON \ - -DARROW_CUDA=OFF \ - -DARROW_DATASET=OFF \ - -DARROW_FILESYSTEM=ON \ - -DARROW_FLIGHT=OFF \ - -DARROW_GANDIVA=OFF \ - -DARROW_HDFS=OFF \ - -DARROW_JSON=OFF \ - -DARROW_ORC=OFF \ - -DARROW_PARQUET=OFF \ - -DARROW_PLASMA=OFF \ - -DARROW_PYTHON=OFF \ - -DARROW_S3=OFF \ - -DARROW_WITH_BZ2=OFF \ - -DARROW_WITH_ZLIB=OFF \ - -DARROW_WITH_LZ4=OFF \ - -DARROW_WITH_SNAPPY=OFF \ - -DARROW_WITH_ZSTD=OFF \ - -DARROW_WITH_BROTLI=OFF \ - -DARROW_IPC=ON \ - -DARROW_BUILD_BENCHMARKS=OFF \ - -DARROW_BUILD_EXAMPLES=OFF \ - -DARROW_BUILD_INTEGRATION=OFF \ - -DARROW_BUILD_UTILITIES=OFF \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_ENABLE_TIMING_TESTS=OFF \ - -DARROW_FUZZING=OFF \ - -DARROW_USE_ASAN=OFF \ - -DARROW_USE_TSAN=OFF \ - -DARROW_USE_UBSAN=OFF \ - -DARROW_JEMALLOC=OFF \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BUILD_STATIC=OFF - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_boost() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/boost/version.hpp" ]]; then - log "boost already installed, skip." - return 0 - fi - - directory="boost_1_74_0" - file="${directory}.tar.gz" - url="https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - ./bootstrap.sh --prefix="${install_prefix}" \ - --with-libraries=system,filesystem,context,program_options,regex,thread,random,chrono,atomic,date_time - ./b2 install link=shared runtime-link=shared variant=release threading=multi - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_openssl() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/openssl/ssl.h" ]]; then - log "openssl already installed, skip." - return 0 - fi - - directory="openssl-OpenSSL_1_1_1k" - file="OpenSSL_1_1_1k.tar.gz" - url="https://github.com/openssl/openssl/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - ./config --prefix="${install_prefix}" -fPIC -shared - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_openssl_static() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/openssl/ssl.h" ]]; then - log "openssl already installed, skip." - return 0 - fi - - directory="openssl-OpenSSL_1_1_1k" - file="OpenSSL_1_1_1k.tar.gz" - url="https://github.com/openssl/openssl/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - ./config --prefix="${install_prefix}" no-shared -fPIC - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_zlib() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/zlib.h" ]]; then - log "zlib already installed, skip." - return 0 - fi - - directory="zlib-1.2.11" - file="v1.2.11.tar.gz" - url="https://github.com/madler/zlib/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DBUILD_SHARED_LIBS=ON - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_protobuf() { - workdir=$1 - install_prefix=$2 - - # Check whether protobuf is installed - if [[ -f "${install_prefix}/include/google/protobuf/port.h" ]]; then - log "protobuf already installed, skip." - return 0 - fi - - # Define the version and download link for protobuf - directory="protobuf-21.9" - file="protobuf-all-21.9.tar.gz" - url="https://github.com/protocolbuffers/protobuf/releases/download/v21.9" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - # Configure and compile protobuf - ./configure --prefix="${install_prefix}" --enable-shared --disable-static - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_grpc() { - workdir=$1 - install_prefix=$2 - - # Check if grpc is installed - if [[ -f "${install_prefix}/include/grpcpp/grpcpp.h" ]]; then - log "grpc already installed, skip." - return 0 - fi - - # Define the grpc version and download link - directory="grpc" - branch="v1.49.1" - file="${directory}-${branch}.tar.gz" - url="https://github.com/grpc/grpc.git" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - if [[ ${url} == *.git ]]; then - clone_if_not_exists ${directory} ${file} "${url}" ${branch} - else - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - fi - pushd ${directory} || exit - - # Configure and compile grpc - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DBUILD_SHARED_LIBS=ON \ - -DgRPC_INSTALL=ON \ - -DgRPC_BUILD_TESTS=OFF \ - -DgRPC_BUILD_CSHARP_EXT=OFF \ - -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ - -DgRPC_BACKWARDS_COMPATIBILITY_MODE=ON \ - -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_ZLIB_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package \ - -DOPENSSL_ROOT_DIR="${install_prefix}" \ - -DCMAKE_CXX_FLAGS="-fpermissive" \ - -DPNG_ARM_NEON_OPT=0 - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_patchelf() { - workdir=$1 - install_prefix=$2 - - # Check if patchelf is installed - if [[ -f "${install_prefix}/bin/patchelf" ]]; then - log "patchelf already installed, skip." - return 0 - fi - - # Define the version and download link for patchelf - ARCH=$(uname -m) - - directory="patchelf" # patchelf doesn't have a folder - file="patchelf-0.14.5-${ARCH}.tar.gz" - url="https://github.com/NixOS/patchelf/releases/download/0.14.5" - url=$(maybe_set_to_cn_url ${url}) - - # Log and start installing patchelf - log "Building and installing ${directory}." - pushd "${workdir}" || exit - mkdir -p "${directory}" - pushd "${directory}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - mkdir -p ${install_prefix}/bin - mv bin/patchelf ${install_prefix}/bin/patchelf - - # Go back to your working directory and clean up your files - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_cppkafka() { - workdir=$1 - install_prefix=$2 - # Check whether cppkafka is installed - if [[ -f "${install_prefix}/include/cppkafka/cppkafka.h" ]]; then - log "cppkafka already installed, skip." - return 0 - fi - - # Define the cppkafka version and download link - directory="cppkafka-0.4.0" - file="0.4.0.tar.gz" - url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" - url=$(maybe_set_to_cn_url ${url}) - - # Log and start installing cppkafka - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - # Configure and compile cppkafka - # cppkafka may not find the lib64 directory - export LIBRARY_PATH=${LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 - - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DCPPKAFKA_DISABLE_TESTS=ON \ - -DCPPKAFKA_DISABLE_EXAMPLES=ON - make -j4 - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_maven() { - workdir=$1 - install_prefix=$2 - - # Check if maven is installed - if [[ -f "${install_prefix}/bin/mvn" ]]; then - log "maven already installed, skip." - return 0 - fi - - # Define the maven version and download link - directory="apache-maven-3.8.6" - file="apache-maven-3.8.6-bin.tar.gz" - url="https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries" - url=$(maybe_set_to_cn_url ${url}) - - # Log and start installing maven - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - cp -r ${directory} "${install_prefix}"/ - - # Configure maven's environment variables - mkdir -p "${install_prefix}"/bin - ln -s "${install_prefix}/${directory}/bin/mvn" "${install_prefix}/bin/mvn" - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_hiactor() { - install_prefix=$1 - pushd /tmp - - git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch - cd hiactor && git submodule update --init --recursive - - # Configure and compile hiactor - sudo bash ./seastar/seastar/install-dependencies.sh - mkdir build && cd build - cmake -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DHiactor_CXX_DIALECT=gnu++17 -DSeastar_CXX_FLAGS="-DSEASTAR_DEFAULT_ALLOCATOR -mno-avx512" .. - make -j 4 && make install - popd && rm -rf /tmp/hiactor -} diff --git a/python/graphscope/gsctl/scripts/lib/install_vineyard.sh b/python/graphscope/gsctl/scripts/lib/install_vineyard.sh deleted file mode 100644 index d7814c1b9616..000000000000 --- a/python/graphscope/gsctl/scripts/lib/install_vineyard.sh +++ /dev/null @@ -1,55 +0,0 @@ -install_vineyard() { - workdir=$1 - install_prefix=$2 - v6d_version=$3 - jobs=${4:-4} # $4:default=4 - V6D_PREFIX="/opt/vineyard" # fixed, related to coordinator/setup.py - - if [[ -f "${V6D_PREFIX}/include/vineyard/client/client.h" ]]; then - log "vineyard already installed, skip." - return 0 - fi - - auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')") - sed -i 's/p.error/logger.warning/g' ${auditwheel_path} - - log "Building and installing v6d." - pushd "${workdir}" || exit - if [[ "${v6d_version}" != "v"* ]]; then - directory="v6d" - file="${directory}-${v6d_version}.tar.gz" - url="https://github.com/v6d-io/v6d.git" - clone_if_not_exists ${directory} "${file}" "${url}" "${v6d_version}" - else - # remove the prefix 'v' - directory="v6d-${v6d_version:1:100}" - file="${directory}.tar.gz" - url="https://github.com/v6d-io/v6d/releases/download/${v6d_version}" - cn_url=$(maybe_set_to_cn_url "${url}") - status=$(curl --head --silent "${cn_url}"/"${file}" | head -n 1) - if echo "${status}" | grep -q 404; then - download_tar_and_untar_if_not_exists ${directory} "${file}" "${url}" - else - download_tar_and_untar_if_not_exists ${directory} "${file}" "${cn_url}" - fi - fi - pushd ${directory} || exit - - # make sure it complain loudly if installing vineyard fails - set -e - cmake . -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DCMAKE_INSTALL_PREFIX="${V6D_PREFIX}" \ - -DBUILD_VINEYARD_TESTS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \ - -DBUILD_VINEYARD_GRAPH_WITH_GAR=ON - make -j"${jobs}" - make install - strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard* - pip3 install --no-cache -i https://pypi.org/simple -U "vineyard" "vineyard-io" - cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/ - set +e - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} diff --git a/python/graphscope/gsctl/scripts/lib/log.sh b/python/graphscope/gsctl/scripts/lib/log.sh deleted file mode 100644 index 4a743eefca47..000000000000 --- a/python/graphscope/gsctl/scripts/lib/log.sh +++ /dev/null @@ -1,20 +0,0 @@ - -err() { - echo -e "$(red_bold "[ERROR] $*")" >&2 -} - -warning() { - echo -e "$(yellow_bold "[WARNING] $*")" >&1 -} - -log() { - echo -e "$(green "$*")" >&1 -} - -info() { - echo -e "$(green "$*")" >&1 -} - -succ() { - echo -e "$(green_bold "[SUCCESS]: $*")" >&1 -} diff --git a/python/graphscope/gsctl/scripts/lib/util.sh b/python/graphscope/gsctl/scripts/lib/util.sh deleted file mode 100644 index 213496f1b8df..000000000000 --- a/python/graphscope/gsctl/scripts/lib/util.sh +++ /dev/null @@ -1,64 +0,0 @@ -# refer from https://github.com/pypa/manylinux/blob/b4884d90c984cb17f7cb4aabe3509347698d7ee7/docker/build_scripts/build_utils.sh#L26 -function fetch_source { - local file=$1 - local url=$2 - if [ -f "${file}" ]; then - echo "${file} exists, skipping fetch" - else - curl -fsSL -o "${file}" "${url}/${file}" - # Use sock5s proxy to download files in case download fails in normal cases - # `host.docker.internal` is the localhost of host machine from a container's perspective. - # See https://docs.docker.com/desktop/networking/#i-want-to-connect-from-a-container-to-a-service-on-the-host - # curl -fsSL -o ${file} ${url}/${file} || curl -x socks5h://host.docker.internal:13659 -fsSL -o ${file} ${url}/${file} - fi -} - -function download_tar_and_untar_if_not_exists { - local directory=$1 - local file=$2 - local url=$3 - if [ ! -d "${directory}" ]; then - [ ! -f "${file}" ] && - fetch_source "${file}" "${url}" - tar zxf "${file}" - fi -} - -function clone_if_not_exists { - local directory=$1 - local file=$2 - local url=$3 - local branch=$4 - if [ ! -d "${directory}" ]; then - if [ ! -f "${file}" ]; then - git clone --depth=1 --branch "${branch}" "${url}" "${directory}" - pushd "${directory}" || exit - git submodule update --init || true - popd || exit - else - tar zxf "${file}" - fi - fi -} - -function cleanup_files { - if [ "${GRAPHSCOPE_NO_INSTALL_CLEANUP}" != "true" ]; then - log "Cleaning up intermediate files [$*]" - log "Disable this behaviour by setting GRAPHSCOPE_NO_INSTALL_CLEANUP=true." - for file in "$@" - do - log "Cleaning up ${file}" - if [[ -f "${file}" || -d "${file}" ]]; then - rm -rf "${file}" - fi - done - fi -} - -function maybe_set_to_cn_url { - local url=$1 - if [ "${GRAPHSCOPE_DOWNLOAD_FROM_CN}" == "true" ]; then - url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" - fi - echo ${url} -} diff --git a/python/graphscope/gsctl/scripts/make_command.sh b/python/graphscope/gsctl/scripts/make_command.sh deleted file mode 100644 index c09b5b16c0a1..000000000000 --- a/python/graphscope/gsctl/scripts/make_command.sh +++ /dev/null @@ -1,107 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh -source ${script_dir}/initialize.sh - - -echo "# this file is located in 'src/make_command.sh'" -echo "# code for 'gs make' goes here" -echo "# you can edit it freely and regenerate (it will not be overwritten)" - -while getopts ":c:i:s:" opt; do - case $opt in - c) - component="$OPTARG" - ;; - i) - install_prefix="$OPTARG" - ;; - s) - storage_type="$OPTARG" - ;; - \?) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - - -log "Making component ${component}" - - -GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" - -echo "dir $GS_SOURCE_DIR" - - -export INSTALL_PREFIX=${install_prefix} - -make_all() { - make all -} - -make_install() { - make install -} - -make_analytical() { - make analytical -} - -make_interactive() { - if [[ ${storage_type} = "experimental" ]]; then - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && make build QUIET_OPT="" - elif [[ ${storage_type} = "vineyard" ]]; then - cd "${GS_SOURCE_DIR}"/interactive_engine && mvn install -DskipTests -Drust.compile.mode=release -P graphscope,graphscope-assembly - cd "${GS_SOURCE_DIR}"/interactive_engine/assembly/target && tar xvzf graphscope.tar.gz - else - make interactive - fi -} - -make_learning() { - make learning -} - -make_analytical-install() { - make analytical-install INSTALL_PREFIX=${install_prefix} -} - -make_interactive-install() { - make interactive-install INSTALL_PREFIX=${install_prefix} -} - -make_learning-install() { - make learning-install INSTALL_PREFIX=${install_prefix} -} - -make_client() { - make client -} - -make_coordinator() { - make coordinator -} - -make_analytical-java() { - make analytical-java -} - -make_analytical-java-install() { - make analytical-java-install INSTALL_PREFIX=${install_prefix} -} - -make_clean() { - make clean -} - -make_${component} diff --git a/python/graphscope/gsctl/scripts/make_image_command.sh b/python/graphscope/gsctl/scripts/make_image_command.sh deleted file mode 100644 index d83ca19b649f..000000000000 --- a/python/graphscope/gsctl/scripts/make_image_command.sh +++ /dev/null @@ -1,95 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh -source ${script_dir}/initialize.sh - - -echo "# this file is located in 'src/make_image_command.sh'" -echo "# code for 'gs make-image' goes here" -echo "# you can edit it freely and regenerate (it will not be overwritten)" - -# allowed: [all, graphscope-dev, coordinator, analytical, analytical-java, interactive, interactive-frontend, interactive-executor, learning, vineyard-dev, vineyard-runtime] - -while getopts ":c:r:t:" opt; do - case $opt in - c) - component="$OPTARG" - ;; - r) - registry="$OPTARG" - ;; - t) - tag="$OPTARG" - ;; - \?) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - - -log "Making image ${component}" - -export INSTALL_PREFIX=${install_prefix} - -make_all() { - cd k8s - make all -} - -make_graphscope-dev() { - cd k8s - make graphscope-dev REGISTRY=${registry} VERSION=${tag} -} - -make_analytical() { - cd k8s - make analytical REGISTRY=${registry} VERSION=${tag} -} - -make_interactive() { - cd k8s - make interactive REGISTRY=${registry} VERSION=${tag} -} - -make_interactive-frontend() { - cd k8s - make interactive-frontend REGISTRY=${registry} VERSION=${tag} -} - -make_interactive-executor() { - cd k8s - make interactive-executor REGISTRY=${registry} VERSION=${tag} -} - -make_learning() { - cd k8s - make learning REGISTRY=${registry} VERSION=${tag} -} - - -make_coordinator() { - cd k8s - make coordinator REGISTRY=${registry} VERSION=${tag} -} - -make_vineyard-dev() { - cd k8s - make vineyard-dev REGISTRY=${registry} VERSION=${tag} -} - -make_vineyard-runtime() { - cd k8s - make vineyard-runtime REGISTRY=${registry} VERSION=${tag} -} - -make_${component} diff --git a/python/graphscope/gsctl/scripts/spark_classpath_command.sh b/python/graphscope/gsctl/scripts/spark_classpath_command.sh deleted file mode 100644 index b763c477a985..000000000000 --- a/python/graphscope/gsctl/scripts/spark_classpath_command.sh +++ /dev/null @@ -1,22 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh - - -echo "# this file is located in 'src/spark_classpath_command.sh'" -echo "# code for 'gs spark-classpath' goes here" -echo "# you can edit it freely and regenerate (it will not be overwritten)" - -# TODO(zhanglei): generate a file named ~/.graphscope_4spark.env - -# check if the file exists -# if yes, raise a warning -# otherwise, generate a new - -echo "$(green_bold "Generated environment variables for spark jobs in ~/.graphscope_4spark.env")" -echo -echo "export the env to your bash terminal by running: source ~/.graphscope_4spark.env" diff --git a/python/graphscope/gsctl/scripts/test_command.sh b/python/graphscope/gsctl/scripts/test_command.sh deleted file mode 100644 index 4b051c24f3df..000000000000 --- a/python/graphscope/gsctl/scripts/test_command.sh +++ /dev/null @@ -1,172 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh -source ${script_dir}/initialize.sh - - -while getopts ":t:d:l:s:k:n:" opt; do - case $opt in - t) - type=$OPTARG - ;; - d) - testdata=$OPTARG - ;; - l) - on_local=$OPTARG - ;; - s) - storage_type=$OPTARG - ;; - k) - on_k8s=$OPTARG - ;; - n) - nx=$OPTARG - ;; - :) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - -export GS_TEST_DIR=${testdata} - -# analytical, analytical-java, interactive, learning, e2e, groot - -GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" - - -function get_test_data { - if [[ ! -d ${GS_TEST_DIR} ]]; then - log "Downloading test data to ${testdata}" - git clone -b master --single-branch --depth=1 https://github.com/graphscope/gstest.git "${GS_TEST_DIR}" - fi -} - -function test_analytical { - get_test_data - info "Testing analytical on local" - "${GS_SOURCE_DIR}"/analytical_engine/test/app_tests.sh --test_dir "${GS_TEST_DIR}" -} - -function test_analytical-java { - get_test_data - info "Testing analytical-java on local" - - pushd "${GS_SOURCE_DIR}"/analytical_engine/java || exit - mvn test -Dmaven.antrun.skip=true - popd || exit - - version=$(cat "${GS_SOURCE_DIR}"/VERSION) - export RUN_JAVA_TESTS=ON - export USER_JAR_PATH="${GS_SOURCE_DIR}"/analytical_engine/java/grape-demo/target/grape-demo-${version}-shaded.jar - # for giraph test - export GIRAPH_JAR_PATH="${GS_SOURCE_DIR}"/analytical_engine/java/grape-giraph/target/grape-giraph-${version}-shaded.jar - - "${GS_SOURCE_DIR}"/analytical_engine/test/app_tests.sh --test_dir "${GS_TEST_DIR}" -} - -function test_interactive { - get_test_data - if [ "${on_local}" == "True" ]; then - if [[ ${storage_type} = "experimental" ]]; then - info "Testing interactive on local with experimental storage" - # IR unit test - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && make test - # CommonType Unit Test - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/common/dyn_type && cargo test - # Store Unit test - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/store/exp_store && cargo test - # IR integration test - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && ./ir_exprimental_ci.sh - # IR integration pattern test - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && ./ir_exprimental_pattern_ci.sh - elif [[ ${storage_type} = "vineyard" ]]; then - info "Testing interactive on local with vineyard storage" - # start vineyard service - export VINEYARD_IPC_SOCKET=/tmp/vineyard.sock - vineyardd --socket=${VINEYARD_IPC_SOCKET} --meta=local & - # load modern graph - export STORE_DATA_PATH="${GS_SOURCE_DIR}"/charts/gie-standalone/data - vineyard-graph-loader --config "${GS_SOURCE_DIR}"/charts/gie-standalone/config/v6d_modern_loader.json - # start gie executor && frontend - export GRAPHSCOPE_HOME="${GS_SOURCE_DIR}"/interactive_engine/assembly/target/graphscope - schema_json=$(ls /tmp/*.json | head -1) - object_id=${schema_json//[^0-9]/} - GRAPHSCOPE_HOME=${GRAPHSCOPE_HOME} ${GRAPHSCOPE_HOME}/bin/giectl create_gremlin_instance_on_local /tmp/gs/${object_id} ${object_id} ${schema_json} 1 1235 1234 8182 ${VINEYARD_IPC_SOCKET} - # IR integration test - sleep 3s - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler - make gremlin_test || true - # clean - rm -rf /tmp/*.json - id=$(pgrep -f 'gaia_executor') - if [[ -n ${id} ]]; then - echo ${id} | xargs kill - fi - id=$(pgrep -f 'frontend') - if [[ -n ${id} ]]; then - echo ${id} | xargs kill - fi - id=$(pgrep -f 'vineyardd') - if [[ -n ${id} ]]; then - echo ${id} | xargs kill -9 - fi - else - info "Testing interactive on local with default storage" - fi - fi - if [ "${on_k8s}" == "True" ]; then - info "Testing interactive on k8s" - export PYTHONPATH="${GS_SOURCE_DIR}"/python:${PYTHONPATH} - cd "${GS_SOURCE_DIR}"/interactive_engine && mvn clean install --quiet -DskipTests -Drust.compile.skip=true -P graphscope,graphscope-assembly - cd "${GS_SOURCE_DIR}"/interactive_engine/tests || exit - ./function_test.sh 8112 2 - fi -} -function test_learning { - get_test_data - err "Not implemented" - exit 1 -} - -function test_e2e { - get_test_data - # Import python projects in the source directory - cd "${GS_SOURCE_DIR}"/python || exit - if [ "${on_local}" == "True" ]; then - # unittest - python3 -m pytest -s -vvv --exitfirst graphscope/tests/minitest/test_min.py - fi - if [ "${on_k8s}" == "True" ]; then - # Run tests in Kubernetes environment using pytest - python3 -m pytest -s -vvv --exitfirst ./graphscope/tests/kubernetes/test_demo_script.py - fi -} - -function test_groot { - # Used to test groot - get_test_data - if [ "${on_local}" == "True" ]; then - info "Testing groot on local" - cd "${GS_SOURCE_DIR}"/interactive_engine/groot-server - mvn test -Pgremlin-test - fi - if [ "${on_k8s}" == "True" ]; then - info "Testing groot on k8s, note you must already setup a groot cluster and necessary environment variables" - cd "${GS_SOURCE_DIR}"/python || exit - python3 -m pytest --exitfirst -s -vvv ./graphscope/tests/kubernetes/test_store_service.py - fi -} - -test_"${type}" diff --git a/python/graphscope/gsctl/utils.py b/python/graphscope/gsctl/utils.py index 4faa36942d76..959695fcd396 100644 --- a/python/graphscope/gsctl/utils.py +++ b/python/graphscope/gsctl/utils.py @@ -70,9 +70,9 @@ def warning(message: str, bold=False): click.secho(message, bold=bold) -def succ(message: str, bold=False): +def succ(message: str, bold=False, fg=None): click.secho("[SUCCESS] ", nl=False, fg="green", bold=True) - click.secho(message, bold=bold) + click.secho(message, bold=bold, fg=fg) def err(message: str, bold=False): diff --git a/python/graphscope/gsctl/version.py b/python/graphscope/gsctl/version.py index 71aa867aef68..1d84f6a3ef0d 100644 --- a/python/graphscope/gsctl/version.py +++ b/python/graphscope/gsctl/version.py @@ -17,15 +17,15 @@ import os -from packaging import version - version_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "VERSION") with open(version_file_path, "r", encoding="utf-8") as fp: - sv = version.parse(fp.read().strip()) - __is_prerelease__ = sv.is_prerelease - __version__ = str(sv) + __version__ = fp.read().strip() -__version_tuple__ = (v for v in __version__.split(".")) +version_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "V6D_VERSION" +) +with open(version_file_path, "r", encoding="utf-8") as fp: + __v6d_version__ = fp.read().strip() del version_file_path diff --git a/python/setup_gsctl.py b/python/setup_gsctl.py index 8bc06513b025..3d118445204f 100644 --- a/python/setup_gsctl.py +++ b/python/setup_gsctl.py @@ -17,6 +17,7 @@ # import os +import platform from setuptools import find_packages # noqa: H301 from setuptools import setup @@ -39,10 +40,12 @@ def parse_version(root, **kwargs): return parse(root, **kwargs) -NAME = "gsctl" -PYTHON_REQUIRES = ">=3.7" REQUIRES = [ - "click >= 8.1.6", + ( + "click" + if platform.system() == "Linux" and platform.machine() == "aarch64" + else "click >= 8.1.6" + ), "graphscope-flex >= 0.27.0", "treelib", "packaging", @@ -50,7 +53,7 @@ def parse_version(root, **kwargs): ] setup( - name=NAME, + name="gsctl", description="Command line tool for GraphScope", author="GraphScope", author_email="graphscope@alibaba-inc.com", @@ -68,7 +71,7 @@ def parse_version(root, **kwargs): long_description="""\ gsctl is a command-line utility for GraphScope. It provides a set of functionalities to make it easy to use GraphScope. These functionalities include building and testing binaries, managing sessions and resources, and more. """, # noqa: E501 - package_data={"graphscope.gsctl": ["scripts/*.sh", "scripts/lib/*.sh", "VERSION"]}, + package_data={"graphscope.gsctl": ["scripts/*.sh", "VERSION", "V6D_VERSION"]}, entry_points={ "console_scripts": [ "gsctl = graphscope.gsctl.gsctl:cli", From 26f5d0a33cf3c173572670e68a3e270ef48a9597 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Fri, 30 Aug 2024 11:56:20 +0800 Subject: [PATCH 2/7] update --- .github/workflows/gae.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml index 7b338e5b0371..b4bb51465346 100644 --- a/.github/workflows/gae.yml +++ b/.github/workflows/gae.yml @@ -48,10 +48,6 @@ jobs: python3 -m pip install --upgrade pip setuptools make coordinator - - name: Setup tmate session - if: false - uses: mxschmitt/action-tmate@v3 - - name: Run Cpp Test env: GS_TEST_DIR: ${{ github.workspace }}/gstest @@ -88,6 +84,10 @@ jobs: python3 -m pytest --cov=graphscope --cov-config=.coveragerc --cov-report=xml \ --cov-report=term --exitfirst -s -v graphscope/tests/unittest/test_java_app.py + - name: Setup tmate session + if: always() + uses: mxschmitt/action-tmate@v3 + - name: Upload Coverage uses: codecov/codecov-action@v3 continue-on-error: true From 5e02d97f9469040ff1c7c8ae52b1145dc352904b Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Fri, 30 Aug 2024 17:49:48 +0800 Subject: [PATCH 3/7] update --- .github/workflows/gae.yml | 4 ++-- python/graphscope/gsctl/scripts/install_deps.sh | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml index b4bb51465346..ba342a325e71 100644 --- a/.github/workflows/gae.yml +++ b/.github/workflows/gae.yml @@ -1,6 +1,6 @@ name: GraphScope GAE CI -on: +mn: workflow_dispatch: push: branches: @@ -85,7 +85,7 @@ jobs: --cov-report=term --exitfirst -s -v graphscope/tests/unittest/test_java_app.py - name: Setup tmate session - if: always() + if: false uses: mxschmitt/action-tmate@v3 - name: Upload Coverage diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index d0bcaa20c70e..9213e3d59bc3 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -100,6 +100,7 @@ while (( "$#" )); do case "$1" in dev) for_analytical=true + for_analytical_java=true for_interactive=true for_learning=true shift @@ -176,9 +177,9 @@ fi # install functions init_workspace_and_env() { - info "creating directory: ${install_prefix} ${MPI_PREFIX} ${tempdir} ${V6D_PREFIX}" - ${SUDO} mkdir -p ${install_prefix} ${MPI_PREFIX} ${tempdir} ${V6D_PREFIX} - ${SUDO} chown -R $(id -u):$(id -g) ${install_prefix} ${MPI_PREFIX} ${tempdir} ${V6D_PREFIX} + info "creating directory: ${install_prefix} ${tempdir} ${V6D_PREFIX}" + ${SUDO} mkdir -p ${install_prefix} ${tempdir} ${V6D_PREFIX} + ${SUDO} chown -R $(id -u):$(id -g) ${install_prefix} ${tempdir} ${V6D_PREFIX} export PATH=${install_prefix}/bin:${PATH} export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 } @@ -406,6 +407,9 @@ install_openmpi() { if [[ -f "${install_prefix}/include/mpi.h" ]]; then return 0 fi + info "creating directory: ${MPI_PREFIX}" + ${SUDO} mkdir -p ${MPI_PREFIX} + ${SUDO} chown -R $(id -u):$(id -g) ${MPI_PREFIX} pushd "${tempdir}" || exit directory="openmpi-4.0.5" file="openmpi-4.0.5.tar.gz" From f6b5018c930df739360dcead9577ba56d62e2c8e Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Mon, 2 Sep 2024 11:38:13 +0800 Subject: [PATCH 4/7] update --- .github/workflows/k8s-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index b9805782fb00..1ca4334b4188 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -455,6 +455,7 @@ jobs: - name: Test demo script of helm installation run: | + sleep 10000000 export NODE_IP=$(kubectl get pod -lgraphscope.coordinator.name=coordinator-graphscope -ojsonpath="{.items[0].status.hostIP}") export NODE_PORT=$(kubectl get services coordinator-service-graphscope -ojsonpath="{.spec.ports[0].nodePort}") echo "GraphScope service listen on ${NODE_IP}:${NODE_PORT}" From 66f692bb7e163efec44cac3932cf851f634e9518 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Mon, 2 Sep 2024 11:44:22 +0800 Subject: [PATCH 5/7] update --- .github/workflows/gae.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml index ba342a325e71..c78d62cc4bb0 100644 --- a/.github/workflows/gae.yml +++ b/.github/workflows/gae.yml @@ -1,6 +1,6 @@ name: GraphScope GAE CI -mn: +on: workflow_dispatch: push: branches: From f9cd9ada5f5ad363d4a97b04c411a9d8037af3bb Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Mon, 2 Sep 2024 19:40:30 +0800 Subject: [PATCH 6/7] update --- k8s/Makefile | 7 +++++++ k8s/dockerfiles/analytical.Dockerfile | 5 +++-- k8s/dockerfiles/coordinator.Dockerfile | 3 ++- k8s/dockerfiles/interactive.Dockerfile | 3 ++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/k8s/Makefile b/k8s/Makefile index 54d9058fec40..63b006e7ac64 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -14,6 +14,8 @@ ARCH := $(subst x86_64,amd64,$(subst aarch64,arm64,$(PLATFORM))) VERSION ?= latest VINEYARD_VERSION ?= $(shell cat $(VINEYARD_VERSION_FILE)) +# Use a dummy builder image (tag=ci, which is actually a busybox) to reduce time and space in CI workflow +BUILDER_VERSION ?= $(VINEYARD_VERSION) PROFILE ?= release CI ?= false # Flex Coordinator @@ -92,6 +94,7 @@ coordinator: --target coordinator \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ + --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg CI=${CI} \ -t graphscope/coordinator:${VERSION} \ @@ -103,6 +106,7 @@ analytical: --target analytical \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ + --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg CI=${CI} \ -t graphscope/analytical:${VERSION} \ @@ -114,6 +118,7 @@ analytical-java: --target analytical-java \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ + --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg CI=${CI} \ -t graphscope/analytical-java:${VERSION} \ @@ -125,6 +130,7 @@ interactive-frontend: --target frontend \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ + --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg profile=$(PROFILE) \ --build-arg CI=${CI} \ @@ -137,6 +143,7 @@ interactive-executor: --target executor \ --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ + --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg VINEYARD_VERSION=$(VINEYARD_VERSION) \ --build-arg profile=$(PROFILE) \ --build-arg CI=${CI} \ diff --git a/k8s/dockerfiles/analytical.Dockerfile b/k8s/dockerfiles/analytical.Dockerfile index e5eea7bd383f..52cf5ab735e6 100644 --- a/k8s/dockerfiles/analytical.Dockerfile +++ b/k8s/dockerfiles/analytical.Dockerfile @@ -2,9 +2,10 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com +ARG BUILDER_VERSION=latest ARG VINEYARD_VERSION=latest ############### BUILDER: ANALYTICAL ####################### -FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false @@ -58,7 +59,7 @@ COPY ./k8s/dockerfiles/entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ############### BUILDER: ANALYTICAL-JAVA ####################### -FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder-java +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder-java COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope diff --git a/k8s/dockerfiles/coordinator.Dockerfile b/k8s/dockerfiles/coordinator.Dockerfile index 6e583dd73fdf..d55086472146 100644 --- a/k8s/dockerfiles/coordinator.Dockerfile +++ b/k8s/dockerfiles/coordinator.Dockerfile @@ -2,8 +2,9 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com +ARG BUILDER_VERSION=latest ARG VINEYARD_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false diff --git a/k8s/dockerfiles/interactive.Dockerfile b/k8s/dockerfiles/interactive.Dockerfile index 04132f13763e..4d1881e84ef9 100644 --- a/k8s/dockerfiles/interactive.Dockerfile +++ b/k8s/dockerfiles/interactive.Dockerfile @@ -2,8 +2,9 @@ ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com +ARG BUILDER_VERSION=latest ARG VINEYARD_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$VINEYARD_VERSION-$ARCH AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false From ba0178026c9048ce88b0a5f4acf9dde57316dbb3 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Mon, 2 Sep 2024 19:42:40 +0800 Subject: [PATCH 7/7] update --- .github/workflows/k8s-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index 1ca4334b4188..b9805782fb00 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -455,7 +455,6 @@ jobs: - name: Test demo script of helm installation run: | - sleep 10000000 export NODE_IP=$(kubectl get pod -lgraphscope.coordinator.name=coordinator-graphscope -ojsonpath="{.items[0].status.hostIP}") export NODE_PORT=$(kubectl get services coordinator-service-graphscope -ojsonpath="{.spec.ports[0].nodePort}") echo "GraphScope service listen on ${NODE_IP}:${NODE_PORT}"