From 7e03336f17121a79e68bca0f9cf0950a30bbc517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 08:55:44 +0800 Subject: [PATCH 01/24] use latest validate-release.yml --- .github/workflows/validate-release.yml | 41 +++++++++++++++++++------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index a45429e88..e338530e8 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -6,6 +6,10 @@ on: release_version: required: true default: '1.0.0' + gpg_user: + required: true + default: 'imbajin' + push: branches: @@ -20,7 +24,7 @@ jobs: env: SCRIPT_PATH: hugegraph-dist/scripts/ URL_PREFIX: https://dist.apache.org/repos/dist/dev/incubator/hugegraph/ - USER: 'imbajin' + USER: ${{ inputs.gpg_user }} # TODO: parse version from the running branch name & also adapt the input version RELEASE_VERSION: '' steps: @@ -104,19 +108,26 @@ jobs: if [[ ! -f "DISCLAIMER" ]]; then echo "The package should include DISCLAIMER file" && exit 1 fi + # 4.2 ensure doesn't contains *GPL/BCL/JSR-275/RSAL/QPL/SSPL/CPOL/NPL1.*/CC-BY + # dependency in LICENSE and NOTICE file + COUNT=$(grep -E "GPL|BCL|JSR-275|RSAL|QPL|SSPL|CPOL|NPL1|CC-BY" LICENSE NOTICE | wc -l) + if [[ $COUNT -ne 0 ]]; then + grep -E "GPL|BCL|JSR-275|RSAL|QPL|SSPL|CPOL|NPL1.0|CC-BY" LICENSE NOTICE + echo "The package shouldn't include GPL* invalid dependency, but get $COUNT" && exit 1 + fi - # 4.2 ensure doesn't contains empty directory or file + # 4.3 ensure doesn't contains empty directory or file COUNT=$(find . -type d -empty | wc -l) if [[ $COUNT -ne 0 ]]; then find . -type d -empty - echo "The package shouldn't include empty directory, but get $COUNT" # TODO: && exit 1 + echo "The package shouldn't include empty directory, but get $COUNT" && exit 1 fi - # 4.3 ensure any file should less than 900kb & not include binary file + # 4.4 ensure any file should less than 900kb & not include binary file COUNT=$(find . -type f -size +900k | wc -l) if [[ $COUNT -ne 0 ]]; then find . -type f -size +900k - echo "The package shouldn't include file larger than 900kb, but get $COUNT" + echo "The package shouldn't include file larger than 900kb, but get $COUNT" && exit 1 fi COUNT=$(find . -type f | perl -lne 'print if -B' | grep -v *.txt | wc -l) if [[ $COUNT -ne 0 ]]; then @@ -124,7 +135,7 @@ jobs: echo "The package shouldn't include binary file, but get $COUNT" fi - # 4.4 test compile the packages + # 4.5 test compile the packages if [[ $JAVA_VERSION == 8 && "$i" =~ "computer" ]]; then cd .. && echo "skip computer module in java8" continue @@ -182,7 +193,7 @@ jobs: fi tar xzvf "$i" || exit - # 7.1 check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "release-docs" dir + # 7.1 check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "licenses" dir cd "$(basename "$i" .tar.gz)" && ls -lh || exit if [[ ! -f "LICENSE" ]]; then echo "The package should include LICENSE file" && exit 1 @@ -193,15 +204,23 @@ jobs: if [[ ! -f "DISCLAIMER" ]]; then echo "The package should include DISCLAIMER file" && exit 1 fi - if [[ ! -d "release-docs" ]]; then - echo "The package should include release-docs dir" && exit 1 + if [[ ! -d "licenses" ]]; then + echo "The package should include licenses dir" && exit 1 + fi + + # 7.2 ensure doesn't contains *GPL/BCL/JSR-275/RSAL/QPL/SSPL/CPOL/NPL1.*/CC-BY + # dependency in LICENSE/NOTICE and licenses/* files + COUNT=$(grep -r -E "GPL|BCL|JSR-275|RSAL|QPL|SSPL|CPOL|NPL1|CC-BY" LICENSE NOTICE licenses | wc -l) + if [[ $COUNT -ne 0 ]]; then + grep -r -E "GPL|BCL|JSR-275|RSAL|QPL|SSPL|CPQL|NPL1|CC-BY" LICENSE NOTICE licenses + echo "The package shouldn't include GPL* invalid dependency, but get $COUNT" && exit 1 fi - # 7.2 ensure doesn't contains empty directory or file + # 7.3 ensure doesn't contains empty directory or file COUNT=$(find . -type d -empty | wc -l) if [[ $COUNT -ne 0 ]]; then find . -type d -empty - echo "The package shouldn't include empty directory, but get $COUNT" # TODO: && exit 1 + echo "The package shouldn't include empty directory, but get $COUNT" && exit 1 fi cd - || exit From 51c53727e14850acdcb23d2ddfd1d9a94735a0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 09:00:30 +0800 Subject: [PATCH 02/24] runs on ubuntu windows macos --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index e338530e8..b4454a666 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -20,7 +20,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: [ubuntu-latest, windows-latest, macos-latest] env: SCRIPT_PATH: hugegraph-dist/scripts/ URL_PREFIX: https://dist.apache.org/repos/dist/dev/incubator/hugegraph/ From b83bfa3861288f750d4cf6328d4b184aa0d961dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 10:53:26 +0800 Subject: [PATCH 03/24] runs on ubuntu windows macos --- .github/workflows/validate-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index b4454a666..df68ff818 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -20,7 +20,8 @@ on: jobs: build: - runs-on: [ubuntu-latest, windows-latest, macos-latest] + name: "Build On ${{ matrix.os }}" + runs-on: ${{ matrix.os }} env: SCRIPT_PATH: hugegraph-dist/scripts/ URL_PREFIX: https://dist.apache.org/repos/dist/dev/incubator/hugegraph/ @@ -269,3 +270,4 @@ jobs: fail-fast: false matrix: java_version: [ '8','11' ] + os: [ubuntu-latest, windows-latest, macos-latest] From ed1c1b65e51efae48ef537e12d9a7de5166751a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 10:55:45 +0800 Subject: [PATCH 04/24] add jdk version in name --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index df68ff818..0d1d841e8 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -20,7 +20,7 @@ on: jobs: build: - name: "Build On ${{ matrix.os }}" + name: "Build On ${{ matrix.os }}-${{ matrix.java_version }}" runs-on: ${{ matrix.os }} env: SCRIPT_PATH: hugegraph-dist/scripts/ From 388c311745a468f297a8217a917f39af72fb033a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 10:59:05 +0800 Subject: [PATCH 05/24] rm windows --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 0d1d841e8..ff84890ab 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -270,4 +270,4 @@ jobs: fail-fast: false matrix: java_version: [ '8','11' ] - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-latest, macos-latest] From d157a30d436db245bae736f74bf3b05dea89a37d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 11:02:11 +0800 Subject: [PATCH 06/24] adjust jdk version --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index ff84890ab..72324e9d7 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -20,7 +20,7 @@ on: jobs: build: - name: "Build On ${{ matrix.os }}-${{ matrix.java_version }}" + name: "Build On ${{ matrix.os }}(jdk-${{ matrix.java_version }})" runs-on: ${{ matrix.os }} env: SCRIPT_PATH: hugegraph-dist/scripts/ From 6a58e1a2d66f745d2f3e8e7e570177f9cb59dbdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 17:25:46 +0800 Subject: [PATCH 07/24] try add centos --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 72324e9d7..478ab460d 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -270,4 +270,4 @@ jobs: fail-fast: false matrix: java_version: [ '8','11' ] - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, centos-latest] From 804932216d7728802dd402715fc10013c8a8dee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 17:43:17 +0800 Subject: [PATCH 08/24] add centos7 --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 478ab460d..eba3036d3 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -270,4 +270,4 @@ jobs: fail-fast: false matrix: java_version: [ '8','11' ] - os: [ubuntu-latest, macos-latest, centos-latest] + os: [ubuntu-latest, macos-latest, centos7] From be7b948b319e4b684ceeb977d0f9b256743c1f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 17:45:47 +0800 Subject: [PATCH 09/24] add todo --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index eba3036d3..b9da282b2 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -270,4 +270,4 @@ jobs: fail-fast: false matrix: java_version: [ '8','11' ] - os: [ubuntu-latest, macos-latest, centos7] + os: [ubuntu-latest, macos-latest, centos7] # TODO: support windows-latest in future From f3841f245b2a6d605b42cce020b3e4fd3d770740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sat, 11 Feb 2023 17:59:25 +0800 Subject: [PATCH 10/24] update validate-release.yml --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index b9da282b2..80cb849c1 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -270,4 +270,4 @@ jobs: fail-fast: false matrix: java_version: [ '8','11' ] - os: [ubuntu-latest, macos-latest, centos7] # TODO: support windows-latest in future + os: [ubuntu-latest, macos-latest] # TODO: support windows-latest or other os in future From 28bcebaf721fd30a31fe2e15c1d821a19d8190ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Sun, 12 Feb 2023 12:12:19 +0800 Subject: [PATCH 11/24] update validate-release.yml --- .github/workflows/validate-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 80cb849c1..347d27ad6 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -270,4 +270,5 @@ jobs: fail-fast: false matrix: java_version: [ '8','11' ] - os: [ubuntu-latest, macos-latest] # TODO: support windows-latest or other os in future + # TODO: support windows-latest or other os in future + os: [ubuntu-latest, macos-latest] From 3b487081b8090a1c9b412864ca0a26742d51eac3 Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 13 Feb 2023 19:42:13 +0800 Subject: [PATCH 12/24] Update validate-release.yml --- .github/workflows/validate-release.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 347d27ad6..9b177e088 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -94,7 +94,7 @@ jobs: echo "$i" # 4.0 check the directory name include "incubating" if [[ ! "$i" =~ "incubating" ]]; then - echo "The package name should include incubating" && exit 1 + echo "The package name $i should include incubating" && exit 1 fi tar xzvf "$i" || exit cd "$(basename "$i" .tar.gz)" || exit @@ -133,6 +133,7 @@ jobs: COUNT=$(find . -type f | perl -lne 'print if -B' | grep -v *.txt | wc -l) if [[ $COUNT -ne 0 ]]; then find . -type f | perl -lne 'print if -B' + # due to the search script is not perfect, we can't exit here (check manually) echo "The package shouldn't include binary file, but get $COUNT" fi @@ -141,7 +142,8 @@ jobs: cd .. && echo "skip computer module in java8" continue fi - mvn package -DskipTests -ntp && ls -lh + mvn package -DskipTests -ntp -e || exit 1 + ls -lh cd .. || exit done @@ -180,6 +182,7 @@ jobs: echo "test hubble" cd ./*hubble*${{ inputs.release_version }} || exit cat conf/hugegraph-hubble.properties && bin/start-hubble.sh + # TODO: need stop the server here cd ../../../ || exit rm -rf ./*src* && ls -lh @@ -190,12 +193,12 @@ jobs: echo "$i" # 7.0 check the directory name include "incubating" if [[ ! "$i" =~ "incubating" ]]; then - echo "The package name should include incubating" && exit 1 + echo "The package name $i should include incubating" && exit 1 fi tar xzvf "$i" || exit - # 7.1 check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "licenses" dir - cd "$(basename "$i" .tar.gz)" && ls -lh || exit + # 7.1 check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" & "licenses" dir + (cd "$(basename "$i" .tar.gz)" && ls -lh) || exit if [[ ! -f "LICENSE" ]]; then echo "The package should include LICENSE file" && exit 1 fi @@ -265,10 +268,11 @@ jobs: cat conf/hugegraph-hubble.properties bin/stop-hubble.sh && bin/start-hubble.sh cd - || exit + # TODO: need stop the server here strategy: fail-fast: false matrix: java_version: [ '8','11' ] - # TODO: support windows-latest or other os in future + # TODO: support windows-latest or other OS in future os: [ubuntu-latest, macos-latest] From 3bdddeb0077b2d9da10f48eb72f51eedd02c1ea4 Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 13 Feb 2023 19:56:58 +0800 Subject: [PATCH 13/24] Update validate-release.yml --- .github/workflows/validate-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 9b177e088..7568a2ecc 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -10,7 +10,6 @@ on: required: true default: 'imbajin' - push: branches: - 'release-*' @@ -20,7 +19,7 @@ on: jobs: build: - name: "Build On ${{ matrix.os }}(jdk-${{ matrix.java_version }})" + name: "Build On ${{ matrix.os }} (java-${{ matrix.java_version }})" runs-on: ${{ matrix.os }} env: SCRIPT_PATH: hugegraph-dist/scripts/ @@ -138,7 +137,7 @@ jobs: fi # 4.5 test compile the packages - if [[ $JAVA_VERSION == 8 && "$i" =~ "computer" ]]; then + if [[ ${{ matrix.java_version }} == 8 && "$i" =~ "computer" ]]; then cd .. && echo "skip computer module in java8" continue fi @@ -198,9 +197,10 @@ jobs: tar xzvf "$i" || exit # 7.1 check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" & "licenses" dir - (cd "$(basename "$i" .tar.gz)" && ls -lh) || exit + cd "$(basename "$i" .tar.gz)" || exit + ls -lh if [[ ! -f "LICENSE" ]]; then - echo "The package should include LICENSE file" && exit 1 + echo "The package should include LICENSE file" && ls -lh && exit 1 fi if [[ ! -f "NOTICE" ]]; then echo "The package should include NOTICE file" && exit 1 From 08ff276388af16b1b2b5ae517eff3f69710d2c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Mon, 13 Feb 2023 21:53:25 +0800 Subject: [PATCH 14/24] stop server after test --- .github/workflows/validate-release.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 7568a2ecc..2245680d9 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -181,9 +181,12 @@ jobs: echo "test hubble" cd ./*hubble*${{ inputs.release_version }} || exit cat conf/hugegraph-hubble.properties && bin/start-hubble.sh - # TODO: need stop the server here cd ../../../ || exit rm -rf ./*src* && ls -lh + # TODO: need stop the server here + cd dist/${{ inputs.release_version }} && ls -lh + cd ./*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit + bin/stop-hugegraph.sh - name: 7. Validate Binary Packages run: | @@ -269,6 +272,9 @@ jobs: bin/stop-hubble.sh && bin/start-hubble.sh cd - || exit # TODO: need stop the server here + cd dist/${{ inputs.release_version }} && ls -lh + cd ./*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit + bin/stop-hugegraph.sh strategy: fail-fast: false From 36678b6a67d6d78e0589dfac6c147c1a6067b4ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Mon, 13 Feb 2023 21:59:05 +0800 Subject: [PATCH 15/24] exit if stop server fail --- .github/workflows/validate-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 2245680d9..914081704 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -186,7 +186,7 @@ jobs: # TODO: need stop the server here cd dist/${{ inputs.release_version }} && ls -lh cd ./*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit - bin/stop-hugegraph.sh + bin/stop-hugegraph.sh || exit - name: 7. Validate Binary Packages run: | @@ -274,7 +274,7 @@ jobs: # TODO: need stop the server here cd dist/${{ inputs.release_version }} && ls -lh cd ./*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit - bin/stop-hugegraph.sh + bin/stop-hugegraph.sh || exit strategy: fail-fast: false From ed900cf115e63ee87d240817f43a2f243bddf5b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Mon, 13 Feb 2023 22:08:12 +0800 Subject: [PATCH 16/24] update validate-release.yml --- .github/workflows/validate-release.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 914081704..553451947 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -181,12 +181,12 @@ jobs: echo "test hubble" cd ./*hubble*${{ inputs.release_version }} || exit cat conf/hugegraph-hubble.properties && bin/start-hubble.sh - cd ../../../ || exit - rm -rf ./*src* && ls -lh # TODO: need stop the server here - cd dist/${{ inputs.release_version }} && ls -lh - cd ./*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit + cd dist/${{ inputs.release_version }}/*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit bin/stop-hugegraph.sh || exit + cd - + cd ../../../ || exit + rm -rf ./*src* && ls -lh - name: 7. Validate Binary Packages run: | @@ -271,9 +271,8 @@ jobs: cat conf/hugegraph-hubble.properties bin/stop-hubble.sh && bin/start-hubble.sh cd - || exit - # TODO: need stop the server here - cd dist/${{ inputs.release_version }} && ls -lh - cd ./*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit + # TODO: need stop the server here + cd dist/${{ inputs.release_version }}/*hugegraph-incubating*${{ inputs.release_version }} || exit bin/stop-hugegraph.sh || exit strategy: From 0d5eb38fd7a8cdeb2f656278e33e011a56e3837d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Mon, 13 Feb 2023 22:50:15 +0800 Subject: [PATCH 17/24] add test log --- .github/workflows/validate-release.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 553451947..890ded9b9 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -182,10 +182,14 @@ jobs: cd ./*hubble*${{ inputs.release_version }} || exit cat conf/hugegraph-hubble.properties && bin/start-hubble.sh # TODO: need stop the server here + echo "====pwd" + pwd cd dist/${{ inputs.release_version }}/*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit bin/stop-hugegraph.sh || exit cd - cd ../../../ || exit + echo "====pwd2" + pwd rm -rf ./*src* && ls -lh - name: 7. Validate Binary Packages @@ -272,6 +276,8 @@ jobs: bin/stop-hubble.sh && bin/start-hubble.sh cd - || exit # TODO: need stop the server here + echo "====pwd3" + pwd cd dist/${{ inputs.release_version }}/*hugegraph-incubating*${{ inputs.release_version }} || exit bin/stop-hugegraph.sh || exit From df3487c53312949e08c5d1e7fe0ef6c4e6700292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Mon, 13 Feb 2023 23:05:08 +0800 Subject: [PATCH 18/24] add test log --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 890ded9b9..8ee352e36 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -184,7 +184,7 @@ jobs: # TODO: need stop the server here echo "====pwd" pwd - cd dist/${{ inputs.release_version }}/*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit + cd ../../../*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit bin/stop-hugegraph.sh || exit cd - cd ../../../ || exit From 63a0cf2d41849e0512b35f560cb796a8acd1cb3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Mon, 13 Feb 2023 23:23:07 +0800 Subject: [PATCH 19/24] stop server after test --- .github/workflows/validate-release.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 8ee352e36..338a7e79c 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -182,14 +182,8 @@ jobs: cd ./*hubble*${{ inputs.release_version }} || exit cat conf/hugegraph-hubble.properties && bin/start-hubble.sh # TODO: need stop the server here - echo "====pwd" - pwd - cd ../../../*hugegraph-incubating*src/*hugegraph*${{ inputs.release_version }} || exit - bin/stop-hugegraph.sh || exit - cd - + jps | grep HugeGraphServer | awk '{print $1}' | xargs kill -9 cd ../../../ || exit - echo "====pwd2" - pwd rm -rf ./*src* && ls -lh - name: 7. Validate Binary Packages @@ -276,10 +270,7 @@ jobs: bin/stop-hubble.sh && bin/start-hubble.sh cd - || exit # TODO: need stop the server here - echo "====pwd3" - pwd - cd dist/${{ inputs.release_version }}/*hugegraph-incubating*${{ inputs.release_version }} || exit - bin/stop-hugegraph.sh || exit + jps | grep HugeGraphServer | awk '{print $1}' | xargs kill -9 strategy: fail-fast: false From 220852fccbf8228b7c5bed2f2d5d9aa720351ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Mon, 13 Feb 2023 23:36:35 +0800 Subject: [PATCH 20/24] fix actions exec fail --- .github/workflows/validate-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 338a7e79c..cac49b59b 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -236,8 +236,6 @@ jobs: cd dist/${{ inputs.release_version }} cd ./*hugegraph-incubating*${{ inputs.release_version }} || exit bin/init-store.sh && sleep 1 - # kill the HugeGraphServer process by jps - jps | grep HugeGraphServer | awk '{print $1}' | xargs kill -9 bin/start-hugegraph.sh && ls ../ cd - || exit From ab015c0c22dae800d9efec162999d2281cda909a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Tue, 14 Feb 2023 07:31:51 +0800 Subject: [PATCH 21/24] update validate-release.yml --- .github/workflows/validate-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index cac49b59b..7a92d85af 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -129,11 +129,11 @@ jobs: find . -type f -size +900k echo "The package shouldn't include file larger than 900kb, but get $COUNT" && exit 1 fi - COUNT=$(find . -type f | perl -lne 'print if -B' | grep -v *.txt | wc -l) + COUNT=$(find . -type f | perl -lne 'print if -B' | grep --exclude=*.{txt} | wc -l) if [[ $COUNT -ne 0 ]]; then find . -type f | perl -lne 'print if -B' # due to the search script is not perfect, we can't exit here (check manually) - echo "The package shouldn't include binary file, but get $COUNT" + echo "The package shouldn't include binary file, but get $COUNT" && exit 1 fi # 4.5 test compile the packages From b0e525d60b41168159f0c9cdd26b1292248f93fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Tue, 14 Feb 2023 11:00:33 +0800 Subject: [PATCH 22/24] fix exclude txt --- .github/workflows/validate-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 7a92d85af..36047eeda 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -129,7 +129,7 @@ jobs: find . -type f -size +900k echo "The package shouldn't include file larger than 900kb, but get $COUNT" && exit 1 fi - COUNT=$(find . -type f | perl -lne 'print if -B' | grep --exclude=*.{txt} | wc -l) + COUNT=$(find . -type f | grep -v .txt | wc -l) if [[ $COUNT -ne 0 ]]; then find . -type f | perl -lne 'print if -B' # due to the search script is not perfect, we can't exit here (check manually) From 5d8b8106bba8a692a11ae5ddd12666d4359f4f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Tue, 14 Feb 2023 13:03:20 +0800 Subject: [PATCH 23/24] fix exclude txt --- .github/workflows/validate-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-release.yml b/.github/workflows/validate-release.yml index 36047eeda..fcff42412 100644 --- a/.github/workflows/validate-release.yml +++ b/.github/workflows/validate-release.yml @@ -129,9 +129,9 @@ jobs: find . -type f -size +900k echo "The package shouldn't include file larger than 900kb, but get $COUNT" && exit 1 fi - COUNT=$(find . -type f | grep -v .txt | wc -l) + COUNT=$(find . -type f | grep -v .txt | perl -lne 'print if -B' | wc -l) if [[ $COUNT -ne 0 ]]; then - find . -type f | perl -lne 'print if -B' + find . -type f | grep -v .txt | perl -lne 'print if -B' # due to the search script is not perfect, we can't exit here (check manually) echo "The package shouldn't include binary file, but get $COUNT" && exit 1 fi From 12908a10bfb7ad7516b0322fbc52477f0dd73ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E5=B9=B4?= <1043706593@qq.com> Date: Fri, 17 Feb 2023 21:03:14 +0800 Subject: [PATCH 24/24] add validate-release.sh --- dist/validate-release.sh | 215 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100755 dist/validate-release.sh diff --git a/dist/validate-release.sh b/dist/validate-release.sh new file mode 100755 index 000000000..735dc30b3 --- /dev/null +++ b/dist/validate-release.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is used to validate the release package, including: +# 1. Check the release package name & content +# 2. Check the release package sha512 & gpg signature +# 3. Compile the source package & run server & toolchain +# 4. Run server & toolchain in binary package + +URL_PREFIX="https://dist.apache.org/repos/dist/dev/incubator/hugegraph/" +# release version (input by committer) +RELEASE_VERSION=$1 +JAVA_VERSION=$2 +USER=$3 +# git release branch (check it carefully) +#GIT_BRANCH="release-${RELEASE_VERSION}" + +RELEASE_VERSION=${RELEASE_VERSION:?"Please input the release version behind script"} + +# step1: download svn files +rm -rf dist/"$RELEASE_VERSION" && svn co ${URL_PREFIX}/"$RELEASE_VERSION" dist/"$RELEASE_VERSION" +cd dist/"$RELEASE_VERSION" || exit + +# step2: check environment & import public keys +shasum --version 1>/dev/null || exit +gpg --version 1>/dev/null || exit + +wget https://downloads.apache.org/incubator/hugegraph/KEYS || exit +gpg --import KEYS +# TODO: how to trust all public keys in gpg list, currently only trust the first one +echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key $USER trust + +echo "trust all pk" +for key in $(gpg --no-tty --list-keys --with-colons | awk -F: '/^pub/ {print $5}'); do + echo -e "5\ny\n" | gpg --batch --command-fd 0 --edit-key "$key" trust +done + +# step3: check sha512 & gpg signature +for i in *.tar.gz; do + echo "$i" + shasum -a 512 --check "$i".sha512 || exit + eval gpg "${GPG_OPT}" --verify "$i".asc "$i" || exit +done + +# step4: validate source packages +ls -lh ./*.tar.gz +for i in *src.tar.gz; do + echo "$i" + #### step4.0: check the directory name include "incubating" + if [[ ! "$i" =~ "incubating" ]]; then + echo "The package name $i should include incubating" && exit 1 + fi + tar xzvf "$i" || exit + cd "$(basename "$i" .tar.gz)" || exit + + #### step4.1: check the directory include "NOTICE" and "LICENSE" file and "DISCLAIMER" file + if [[ ! -f "LICENSE" ]]; then + echo "The package $i should include LICENSE file" && exit 1 + fi + if [[ ! -f "NOTICE" ]]; then + echo "The package $i should include NOTICE file" && exit 1 + fi + if [[ ! -f "DISCLAIMER" ]]; then + echo "The package $i should include DISCLAIMER file" && exit 1 + fi + # step4.2: ensure doesn't contains *GPL/BCL/JSR-275/RSAL/QPL/SSPL/CPOL/NPL1.*/CC-BY + COUNT=$(grep -E "GPL|BCL|JSR-275|RSAL|QPL|SSPL|CPOL|NPL1|CC-BY" LICENSE NOTICE | wc -l) + if [[ $COUNT -ne 0 ]]; then + grep -E "GPL|BCL|JSR-275|RSAL|QPL|SSPL|CPOL|NPL1.0|CC-BY" LICENSE NOTICE + echo "The package $i shouldn't include GPL* invalid dependency, but get $COUNT" && exit 1 + fi + # step4.3: ensure doesn't contains empty directory or file + COUNT=$(find . -type d -empty | wc -l) + if [[ $COUNT -ne 0 ]]; then + find . -type d -empty + echo "The package $i should not include empty directory, but get $COUNT" # TODO: && exit 1 + fi + # step4.4: ensure any file should less than 900kb & not include binary file + COUNT=$(find . -type f -size +900k | wc -l) + if [[ $COUNT -ne 0 ]]; then + find . -type f -size +900k + echo "The package $i shouldn't include file larger than 900kb, but get $COUNT" && exit 1 + fi + COUNT=$(find . -type f | perl -lne 'print if -B' | grep -v *.txt | wc -l) + if [[ $COUNT -ne 0 ]]; then + find . -type f | perl -lne 'print if -B' + echo "The package $i shouldn't include binary file, but get $COUNT" + fi + + #### step4.5: test compile the packages + if [[ $JAVA_VERSION == 8 && "$i" =~ "computer" ]]; then + cd .. && echo "skip computer module in java8" + continue + fi + mvn package -DskipTests -ntp && ls -lh + cd .. || exit +done + +#### step5: run the compiled packages in server +ls -lh +cd ./*hugegraph-incubating*src/*hugegraph*"${RELEASE_VERSION}" || exit +bin/init-store.sh && sleep 1 +bin/start-hugegraph.sh && ls ../../ +cd ../../ || exit + +#### step6: run the compiled packages in toolchain (include loader/tool/hubble) +cd ./*toolchain*src || exit +ls -lh +cd ./*toolchain*"${RELEASE_VERSION}" || exit +ls -lh + +##### step6.1: test loader +cd ./*loader*"${RELEASE_VERSION}" || exit +bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy \ + -g hugegraph || exit +cd .. || exit + +##### step6.2: test tool +cd ./*tool*"${RELEASE_VERSION}" || exit +bin/hugegraph gremlin-execute --script 'g.V().count()' || exit +bin/hugegraph task-list || exit +bin/hugegraph backup -t all --directory ./backup-test || exit +cd .. || exit + +##### step6.3: test hubble +cd ./*hubble*"${RELEASE_VERSION}" || exit +# TODO: add hubble doc & test it +cat conf/hugegraph-hubble.properties && bin/start-hubble.sh +cd ../../../ || exit + +# step7: validate the binary packages +rm -rf ./*src* && ls -lh +for i in *.tar.gz; do + echo "$i" + #### step7.1: check the directory name include "incubating" + if [[ ! "$i" =~ "incubating" ]]; then + echo "The package name $i should include incubating" && exit 1 + fi + tar xzvf "$i" || exit + + #### step7.2: check root dir include "NOTICE"/"LICENSE"/"DISCLAIMER" files & "licenses" dir + cd "$(basename "$i" .tar.gz)" && ls -lh || exit + if [[ ! -f "LICENSE" ]]; then + echo "The package $i should include LICENSE file" && exit 1 + fi + if [[ ! -f "NOTICE" ]]; then + echo "The package $i should include NOTICE file" && exit 1 + fi + if [[ ! -f "DISCLAIMER" ]]; then + echo "The package $i should include DISCLAIMER file" && exit 1 + fi + if [[ ! -d "licenses" ]]; then + echo "The package $i should include licenses dir" && exit 1 + fi + #### step7.3: ensure doesn't contains *GPL/BCL/JSR-275/RSAL/QPL/SSPL/CPOL/NPL1.*/CC-BY + COUNT=$(grep -r -E "GPL|BCL|JSR-275|RSAL|QPL|SSPL|CPOL|NPL1|CC-BY" LICENSE NOTICE licenses | wc -l) + if [[ $COUNT -ne 0 ]]; then + grep -r -E "GPL|BCL|JSR-275|RSAL|QPL|SSPL|CPQL|NPL1|CC-BY" LICENSE NOTICE licenses + echo "The package $i shouldn't include GPL* invalid dependency, but get $COUNT" && exit 1 + fi + #### step7.4: ensure doesn't contains empty directory or file + COUNT=$(find . -type d -empty | wc -l) + if [[ $COUNT -ne 0 ]]; then + find . -type d -empty + echo "The package $i should not include empty directory, but get $COUNT" # TODO: && exit 1 + fi + cd - || exit +done + +#### step8: start the server +cd ./*hugegraph-incubating*"${RELEASE_VERSION}" || exit +bin/init-store.sh && sleep 1 +# kill the HugeGraphServer process by jps +jps | grep HugeGraphServer | awk '{print $1}' | xargs kill -9 +bin/start-hugegraph.sh && ls ../ +cd - || exit + +#### step9: running toolchain +cd ./*toolchain*"${RELEASE_VERSION}" || exit +ls -lh +##### step9.1: test loader +cd ./*loader*"${RELEASE_VERSION}" || exit +bin/hugegraph-loader.sh -f ./example/file/struct.json -s ./example/file/schema.groovy \ + -g hugegraph || exit +cd - || exit + +##### step9.2: test tool +cd ./*tool*"${RELEASE_VERSION}" || exit +bin/hugegraph gremlin-execute --script 'g.V().count()' || exit +bin/hugegraph task-list || exit +bin/hugegraph backup -t all --directory ./backup-test || exit +cd - || exit + +##### step9.3: test hubble +cd ./*hubble*"${RELEASE_VERSION}" || exit +# TODO: add hubble doc & test it +cat conf/hugegraph-hubble.properties +bin/stop-hubble.sh && bin/start-hubble.sh +cd - || exit + +echo "Finish validate, please check all steps manually again!"