From bdf34b3b717c53abec655cef234815fc2b5fec16 Mon Sep 17 00:00:00 2001 From: ykadowak Date: Thu, 30 Nov 2023 02:33:43 +0000 Subject: [PATCH] Revert "[CI]: Refactor github actions (#2141)" This reverts commit af8d98b741d983e3617f2c3eeedb63a93a94f896. --- .github/actions/deploy-chaos-mesh/action.yaml | 50 -- .../detect-docker-image-tags/action.yaml | 13 +- .../determine-docker-image-tag/action.yaml | 42 -- .github/actions/docker-build/action.yaml | 50 +- .github/actions/dump-context/action.yaml | 41 -- .../e2e-deploy-vald-helm-operator/action.yaml | 74 +-- .github/actions/e2e-deploy-vald/action.yaml | 67 ++- .github/actions/notify-slack/action.yaml | 32 -- .github/actions/scan-docker-image/action.yaml | 35 -- .github/actions/setup-e2e/action.yaml | 86 --- .github/actions/setup-go/action.yaml | 30 -- .github/actions/setup-helm/action.yaml | 30 -- .github/actions/setup-k3d/action.yaml | 71 --- .github/actions/setup-yq/action.yaml | 34 -- .../actions/wait-for-docker-image/action.yaml | 4 +- .github/chatops_commands.md | 2 + .github/helm/values/values-chaos.yaml | 21 +- .github/valdrelease/valdrelease.yaml | 2 - .github/workflows/_docker-image-scan.yaml | 35 -- .github/workflows/_docker-image.yaml | 94 ---- .github/workflows/_update-protobuf.yaml | 37 -- .github/workflows/build-binaries.yml | 77 ++- .github/workflows/build-protobuf.yml | 41 +- .github/workflows/chatops-help.yml | 36 +- .github/workflows/chatops.yml | 205 ++++++-- .github/workflows/codeql-analysis.yml | 49 +- .github/workflows/coverage.yml | 42 +- .../detect-internal-config-changes.yml | 31 +- .github/workflows/dockers-agent-ngt-image.yml | 123 ++++- .../workflows/dockers-agent-sidecar-image.yml | 123 ++++- .../workflows/dockers-ci-container-image.yml | 88 +++- .../workflows/dockers-dev-container-image.yml | 86 ++- .../dockers-discoverer-k8s-image.yml | 123 ++++- .../dockers-gateway-filter-image.yml | 130 ++++- .../workflows/dockers-gateway-lb-image.yml | 123 ++++- .../workflows/dockers-helm-operator-image.yml | 123 ++++- .github/workflows/dockers-image-scan.yml | 317 +++++++++-- .github/workflows/dockers-loadtest-image.yml | 125 ++++- .../workflows/dockers-manager-index-image.yml | 123 ++++- .github/workflows/e2e-chaos.yaml | 442 +++++++++++----- .github/workflows/e2e-code-bench-agent.yaml | 70 ++- .github/workflows/e2e-deploy.yml | 495 ++++++++++++++++++ .github/workflows/e2e-max-dim.yml | 105 +++- .github/workflows/e2e-profiling.yml | 112 ++-- .github/workflows/e2e.yml | 288 ---------- .github/workflows/format.yml | 54 +- .github/workflows/fossa.yml | 39 +- .github/workflows/helm-lint.yml | 98 +++- .github/workflows/helm.yml | 118 +++-- .github/workflows/labeler.yml | 39 +- .github/workflows/reviewdog-hadolint.yml | 38 +- .github/workflows/reviewdog-k8s.yml | 52 +- .github/workflows/reviewdog-markdown.yml | 55 +- .github/workflows/reviewdog.yml | 41 +- .github/workflows/semver.yml | 41 +- .github/workflows/test-hack.yml | 66 ++- .../workflows/{unit-test.yaml => test.yml} | 64 ++- .github/workflows/update-protobuf.yaml | 55 -- .github/workflows/update-protobuf.yml | 85 +++ ...pdate_pull_request_and_issue_template.yml} | 38 +- Makefile | 4 - Makefile.d/k8s.mk | 1 - .../agent/core/ngt/service/ngt_e2s_test.go | 12 +- 63 files changed, 3496 insertions(+), 1731 deletions(-) delete mode 100644 .github/actions/deploy-chaos-mesh/action.yaml delete mode 100644 .github/actions/determine-docker-image-tag/action.yaml delete mode 100644 .github/actions/dump-context/action.yaml delete mode 100644 .github/actions/notify-slack/action.yaml delete mode 100644 .github/actions/scan-docker-image/action.yaml delete mode 100644 .github/actions/setup-e2e/action.yaml delete mode 100644 .github/actions/setup-go/action.yaml delete mode 100644 .github/actions/setup-helm/action.yaml delete mode 100644 .github/actions/setup-k3d/action.yaml delete mode 100644 .github/actions/setup-yq/action.yaml delete mode 100644 .github/workflows/_docker-image-scan.yaml delete mode 100644 .github/workflows/_docker-image.yaml delete mode 100644 .github/workflows/_update-protobuf.yaml create mode 100644 .github/workflows/e2e-deploy.yml delete mode 100644 .github/workflows/e2e.yml rename .github/workflows/{unit-test.yaml => test.yml} (70%) delete mode 100644 .github/workflows/update-protobuf.yaml create mode 100644 .github/workflows/update-protobuf.yml rename .github/workflows/{update-pull-request-and-issue-template.yml => update_pull_request_and_issue_template.yml} (82%) diff --git a/.github/actions/deploy-chaos-mesh/action.yaml b/.github/actions/deploy-chaos-mesh/action.yaml deleted file mode 100644 index 41de55f9d0..0000000000 --- a/.github/actions/deploy-chaos-mesh/action.yaml +++ /dev/null @@ -1,50 +0,0 @@ -name: "Deploy Chaos Mesh" -description: "A action to deploy Chaos Mesh" - -inputs: - chaos_mesh_version: - description: "The Chaos Mesh version to use. The default version is `versions/CHAOS_MESH_VERSION`" - required: false - default: "" - helm_extra_options: - description: "Extra options that passed to Helm command" - required: false - default: "" - -runs: - using: "composite" - steps: - - name: Fetch Chaos Mesh version - id: chaos_mesh_version - shell: bash - run: | - CHAOS_MESH_VERSION=${CHAOS_MESH_VERSION:-`cat versions/CHAOS_MESH_VERSION`} - - echo "version=${CHAOS_MESH_VERSION}" >> $GITHUB_OUTPUT - env: - CHAOS_MESH_VERSION: ${{ inputs.chaos_mesh_version }} - - - name: Check k3d - shell: bash - id: check_k3d - run: | - kubectl cluster-info - - - name: Check Helm - shell: bash - run: | - helm version - - - name: Deploy Chaos Mesh - shell: bash - run: | - make kubectl/install - - curl -sSL "https://mirrors.chaos-mesh.org/v${CHAOS_MESH_VERSION}/install.sh" | bash -s -- --k3s - - helm install \ - ${HELM_EXTRA_OPTIONS} \ - vald-chaos-test tests/chaos/chart - env: - HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} - CHAOS_MESH_VERSION: ${{ steps.chaos_mesh_version.outputs.version }} diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml index 01f9c8891a..75e278d2c7 100644 --- a/.github/actions/detect-docker-image-tags/action.yaml +++ b/.github/actions/detect-docker-image-tags/action.yaml @@ -14,15 +14,14 @@ # limitations under the License. # name: "Detect Docker image tags" -description: "A action to detect Docker image tags" - +description: "Detect Docker image tags" inputs: tag_name: - description: "Tag name to check whether exists or not" + description: "tag name to check whether exists or not" required: true default: "" images: - description: "Image names" + description: "image names" required: false default: "vdaas/vald-agent-ngt \ vdaas/vald-discoverer-k8s \ @@ -30,12 +29,11 @@ inputs: vdaas/vald-manager-index" outputs: HELM_EXTRA_OPTIONS: - description: "Helm extra options that specifies image tags" + description: "helm extra options that specifies image tags" value: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} IMAGE_TAGS: - description: "Specifies image tags" + description: "specifies image tags" value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }} - runs: using: "composite" steps: @@ -49,7 +47,6 @@ runs: ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag" ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag" ["vdaas/vald-manager-index"]="manager.index.image.tag" - ["vdaas/vald-helm-operator"]="image.tag" ) for image in ${IMAGES} diff --git a/.github/actions/determine-docker-image-tag/action.yaml b/.github/actions/determine-docker-image-tag/action.yaml deleted file mode 100644 index 177109f964..0000000000 --- a/.github/actions/determine-docker-image-tag/action.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "Determine Docker image tag" -description: "A action to determine Docker image tag" - -outputs: - PRIMARY_TAG: - description: "Primary tag" - value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} - -runs: - using: "composite" - steps: - - name: Dump Github context - shell: bash - run: | - echo "GITHUB_REF $GITHUB_REF" - echo "GITHUB_EVENT_PATH $GITHUB_EVENT_PATH" - echo "GITHUB_EVENT_NAME ${{ github.event_name }}" - echo "GITHUB_EVENT_NUMBER ${{ github.event.number }}" - - - name: Determine tag name - shell: bash - id: determine_tag_name - run: | - if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then - tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'` - primary_tag="${tag_name}" - elif [ "${{ github.event_name }}" = "pull_request" ]; then - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR-${pr_num}" > versions/VALD_VERSION - primary_tag="pr-${pr_num}" - elif [ "${{ github.event_name }}" = "pull_request_target" ]; then - pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` - echo "PR-${pr_num}" > versions/VALD_VERSION - primary_tag="pr-${pr_num}" - elif [ "$GITHUB_REF" = "refs/heads/main" ]; then - echo "nightly" > versions/VALD_VERSION - primary_tag="nightly" - else - primary_tag="unknown" - fi - echo "PRIMARY_TAG is determined: ${primary_tag}" - echo "PRIMARY_TAG=${primary_tag}" >> $GITHUB_OUTPUT diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml index 1817c593df..b97b4b49af 100644 --- a/.github/actions/docker-build/action.yaml +++ b/.github/actions/docker-build/action.yaml @@ -14,38 +14,36 @@ # limitations under the License. # name: "Build Docker images" -description: "A action to build Docker images and publish them" - +description: "Build Docker images and publish them" inputs: target: - description: "Build target" + description: "build target" required: true default: "base" builder: - description: "Buildx builder name" + description: "buildx builder name" required: true default: "" platforms: - description: "If it is specified, specified platforms will be used." + description: "if it is specified, specified platforms will be used." required: false default: "" outputs: IMAGE_NAME: - description: "Image name" + description: "image name" value: ${{ steps.image_name.outputs.IMAGE_NAME }} ALTER_IMAGE_NAME: - description: "Alter image name" + description: "alter image name" value: ${{ steps.image_name.outputs.ALTER_IMAGE_NAME }} PRIMARY_TAG: - description: "Primary tag" + description: "primary tag" value: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} PLATFORMS: - description: "Target platforms" + description: "target platforms" value: ${{ steps.determine_platforms.outputs.PLATFORMS }} EXTRA_TAGS: - description: "Extra tags" + description: "extra tags" value: ${{ steps.add_extra_tags.outputs.EXTRA_TAGS }} - runs: using: "composite" steps: @@ -62,11 +60,33 @@ runs: echo "ALTER_IMAGE_NAME=${alter_image_name}" >> $GITHUB_OUTPUT env: TARGET: ${{ inputs.target }} - - name: Determine tag name + shell: bash id: determine_tag_name - uses: ./.github/actions/determine-docker-image-tag - + run: | + echo "GITHUB_REF $GITHUB_REF" + echo "GITHUB_EVENT_PATH $GITHUB_EVENT_PATH" + echo "GITHUB_EVENT_NAME ${{ github.event_name }}" + echo "GITHUB_EVENT_NUMBER ${{ github.event.number }}" + if [[ "$GITHUB_REF" =~ ^refs/tags/.* ]]; then + tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'` + primary_tag="${tag_name}" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION + primary_tag="pr-${pr_num}" + elif [ "${{ github.event_name }}" = "pull_request_target" ]; then + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR-${pr_num}" > versions/VALD_VERSION + primary_tag="pr-${pr_num}" + elif [ "$GITHUB_REF" = "refs/heads/main" ]; then + echo "nightly" > versions/VALD_VERSION + primary_tag="nightly" + else + primary_tag="unknown" + fi + echo "PRIMARY_TAG is determined: ${primary_tag}" + echo "PRIMARY_TAG=${primary_tag}" >> $GITHUB_OUTPUT - name: Determine platforms shell: bash id: determine_platforms @@ -86,7 +106,6 @@ runs: echo "PLATFORMS=${platforms}" >> $GITHUB_OUTPUT env: TARGET_PLATFORMS: ${{ inputs.platforms }} - - name: Add extra tags shell: bash id: add_extra_tags @@ -102,7 +121,6 @@ runs: IMAGE_NAME: ${{ steps.image_name.outputs.IMAGE_NAME }} ALTER_IMAGE_NAME: ${{ steps.image_name.outputs.ALTER_IMAGE_NAME }} PRIMARY_TAG: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} - - name: Build and Push shell: bash id: build_and_push diff --git a/.github/actions/dump-context/action.yaml b/.github/actions/dump-context/action.yaml deleted file mode 100644 index 33e3415d39..0000000000 --- a/.github/actions/dump-context/action.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: "Dump context to log" -description: "A action to dump context to log" - -runs: - using: "composite" - steps: - - name: "Dump GitHub context" - shell: bash - run: echo $JSON - env: - JSON: ${{toJSON(github)}} - - - name: "Dump job context" - shell: bash - run: echo $JSON - env: - JSON: ${{ toJSON(job) }} - - - name: "Dump steps context" - shell: bash - run: echo $JSON - env: - JSON: ${{toJSON(steps)}} - - - name: "Dump runner context" - shell: bash - run: echo $JSON - env: - JSON: ${{toJSON(runner)}} - - - name: "Dump strategy context" - shell: bash - run: echo $JSON - env: - JSON: ${{toJSON(strategy)}} - - - name: "Dump matrix context" - shell: bash - run: echo $JSON - env: - JSON: ${{toJSON(matrix)}} diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml index 09dc80faef..ad5848cf23 100644 --- a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml +++ b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml @@ -13,28 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Deploy Vald using vald-helm-operator for E2E test" -description: "A action to deploy vald using vald-helm-operator for E2E test" - +name: "Setting E2E test environment" +description: "A workflow to set E2E test environment" inputs: + require_libhdf5: + description: "If libhdf5 is required, set this to true." + required: false + default: "false" require_minio: description: "If Minio is required, set this to true." required: false default: "false" - helm_extra_options: - description: "Extra options that passed to Helm command to deploy vald helm operator" - required: false - default: "" valdrelease: - description: "Path to the valdrelease.yaml that apply to cluster" + description: "path to the valdrelease.yaml that apply to cluster." required: true default: "true" wait_for_selector: - description: "Label selector used for specifying a pod waited for" + description: "label selector used for specifying a pod waited for" required: false default: "app=vald-lb-gateway" wait_for_timeout: - description: "Timeout used for waiting for pods" + description: "timeout used for waiting for pods" required: false default: "600s" use_local_charts: @@ -43,49 +42,62 @@ inputs: default: "true" outputs: POD_NAME: - description: "A pod name that waited for" + description: "a pod name that waited for" value: ${{ steps.deploy_vald.outputs.POD_NAME }} - runs: using: "composite" steps: - - name: Deploy Minio - id: deploy_minio + - name: Install libhdf5 + id: install_libhdf5 shell: bash - if: ${{ inputs.require_minio == 'true' }} run: | - make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - - - name: Dump valdrelease + if [[ "${REQUIRE_LIBHDF5}" == "true" ]]; then + sudo apt-get update + sudo apt-get install -y libhdf5-dev + else + echo "skipped." + fi + env: + REQUIRE_LIBHDF5: ${{ inputs.require_libhdf5 }} + - name: check k3d shell: bash + id: check_k3d run: | - cat ${{ inputs.valdrelease }} - - - name: Deploy vald helm operator from remote charts + kubectl cluster-info + - name: Helm version + shell: bash + id: helm_version + run: | + helm version + - name: deploy Minio + id: deploy_minio + shell: bash + run: | + if [[ "${REQUIRE_MINIO}" == "true" ]]; then + make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy + else + echo "skipped." + fi + env: + REQUIRE_MINIO: ${{ inputs.require_minio }} + - name: deploy vald helm operator from remote charts shell: bash id: deploy_vald_helm_operator_remote if: ${{ inputs.use_local_charts == 'false' }} run: | helm install vald-helm-operator \ --set image.tag=nightly \ - ${HELM_EXTRA_OPTIONS} \ charts/vald-helm-operator/. sleep 3 - env: - HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} - - - name: Deploy vald helm operator from local charts + - name: deploy vald helm operator from local charts shell: bash id: deploy_vald_helm_operator_local if: ${{ inputs.use_local_charts == 'true' }} run: | - make k8s/vald-helm-operator/deploy VERSION=nightly HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" + make k8s/vald-helm-operator/deploy sleep 3 - env: - HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} - - - name: Deploy vald + - name: deploy vald shell: bash id: deploy_vald run: | diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml index 6a7c2ec8ab..92cb42245a 100644 --- a/.github/actions/e2e-deploy-vald/action.yaml +++ b/.github/actions/e2e-deploy-vald/action.yaml @@ -13,28 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Deploy Vald for E2E test" -description: "A action to deploy vald for E2E test" - +name: "Setting E2E test environment" +description: "A workflow to set E2E test environment" inputs: + require_libhdf5: + description: "If libhdf5 is required, set this to true." + required: false + default: "false" require_minio: description: "If Minio is required, set this to true." required: false default: "false" helm_extra_options: - description: "Extra options that passed to Helm command." + description: "extra options that passed to Helm command." required: false default: "" values: - description: "Path to the values.yaml that passed to Helm command." + description: "path to the values.yaml that passed to Helm command." required: false default: "false" wait_for_selector: - description: "Label selector used for specifying a pod waited for" + description: "label selector used for specifying a pod waited for" required: false default: "app=vald-lb-gateway" wait_for_timeout: - description: "Timeout used for waiting for pods" + description: "timeout used for waiting for pods" required: false default: "600s" use_local_charts: @@ -43,25 +46,45 @@ inputs: default: "true" outputs: POD_NAME: - description: "A pod name that waited for" + description: "a pod name that waited for" value: ${{ steps.get_real_pod_name.outputs.POD_NAME }} - runs: using: "composite" steps: - - name: Deploy Minio - id: deploy_minio + - name: Install libhdf5 + id: install_libhdf5 shell: bash - if: ${{ inputs.require_minio == 'true' }} run: | - make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy - - - name: Dump Helm values + if [[ "${REQUIRE_LIBHDF5}" == "true" ]]; then + sudo apt-get update + sudo apt-get install -y libhdf5-dev + else + echo "skipped." + fi + env: + REQUIRE_LIBHDF5: ${{ inputs.require_libhdf5 }} + - name: check k3d shell: bash + id: check_k3d run: | - cat ${{ inputs.values }} - - - name: Deploy vald from remote charts + kubectl cluster-info + - name: Helm version + shell: bash + id: helm_version + run: | + helm version + - name: deploy Minio + id: deploy_minio + shell: bash + run: | + if [[ "${REQUIRE_MINIO}" == "true" ]]; then + make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy + else + echo "skipped." + fi + env: + REQUIRE_MINIO: ${{ inputs.require_minio }} + - name: deploy vald from remote charts shell: bash id: deploy_vald_remote if: ${{ inputs.use_local_charts == 'false' }} @@ -85,13 +108,12 @@ runs: HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }} WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }} - - - name: Deploy vald from local charts + - name: deploy vald from local charts shell: bash id: deploy_vald_local if: ${{ inputs.use_local_charts == 'true' }} run: | - make k8s/vald/deploy VERSION=nightly HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" + make k8s/vald/deploy HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" sleep 3 @@ -106,8 +128,7 @@ runs: HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }} WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }} - - - name: Get real pod name + - name: get real pod name shell: bash id: get_real_pod_name env: diff --git a/.github/actions/notify-slack/action.yaml b/.github/actions/notify-slack/action.yaml deleted file mode 100644 index a386e50f8c..0000000000 --- a/.github/actions/notify-slack/action.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: "Notify Slack" -description: "A action to notify Slack" - -inputs: - author_name: - description: "User name for slack notification" - required: false - token: - description: "Github Token" - default: ${{ github.token }} - required: false - slack_notify_webhook_url: - description: "Slack Notify Webhook URL" - required: true - status: - description: "Specify success or failure or cancelled or custom" - required: false - -runs: - using: "composite" - steps: - - uses: technote-space/workflow-conclusion-action@v2 - with: - GITHUB_TOKEN: ${{ inputs.token }} - - - uses: 8398a7/action-slack@v3 - with: - author_name: ${{ inputs.author_name }} - status: ${{ inputs.status != '' && inputs.status || env.WORKFLOW_CONCLUSION }} - token: ${{ inputs.token }} - env: - SLACK_WEBHOOK_URL: ${{ inputs.slack_notify_webhook_url }} diff --git a/.github/actions/scan-docker-image/action.yaml b/.github/actions/scan-docker-image/action.yaml deleted file mode 100644 index d900db91be..0000000000 --- a/.github/actions/scan-docker-image/action.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Scan the Docker image" -description: "A action to scan the Docker image" - -inputs: - image_ref: - description: "Docker image reference" - required: true - default: "" - severity: - description: "Severities of vulnerabilities to be displayed" - required: false - default: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL" - -runs: - using: "composite" - steps: - - name: Run vulnerability scanner (table) - uses: aquasecurity/trivy-action@master - with: - image-ref: ${{ inputs.image_ref }} - format: "table" - severity: ${{ inputs.severity }} - - - name: Run vulnerability scanner (sarif) - uses: aquasecurity/trivy-action@master - with: - image-ref: ${{ inputs.image_ref }} - format: "sarif" - output: "trivy-results.sarif" - severity: ${{ inputs.severity }} - - - name: Upload Trivy scan results to Security tab - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: "trivy-results.sarif" diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml deleted file mode 100644 index 2f0ffbffba..0000000000 --- a/.github/actions/setup-e2e/action.yaml +++ /dev/null @@ -1,86 +0,0 @@ -name: "Setup E2E environment" -description: "A action to set up the environment for executing E2E test" - -inputs: - require_libhdf5: - description: "If libhdf5 is not required, set this to false" - required: false - default: "true" - require_go: - description: "If go is not required, set this to false" - required: false - default: "true" - require_helm: - description: "If helm is not required, set this to false" - required: false - default: "true" - require_k3d: - description: "If k3d is not required, set this to false" - required: false - default: "true" - ingress_port: - description: 'If it is not "0", ingress will be exposed to the specified port' - required: false - default: "0" - target_images: - description: "Image names" - required: false - default: "vdaas/vald-agent-ngt \ - vdaas/vald-discoverer-k8s \ - vdaas/vald-lb-gateway \ - vdaas/vald-manager-index" - -outputs: - HELM_EXTRA_OPTIONS: - description: "Helm extra options that specifies E2E target image tags" - value: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} - IMAGE_TAGS: - description: "Specifies E2E target image tags" - value: ${{ steps.specify_container_versions.outputs.IMAGE_TAGS }} - -runs: - using: "composite" - steps: - - name: Install libhdf5 - if: ${{ inputs.require_libhdf5 == 'true' }} - shell: bash - run: | - sudo apt-get update - sudo apt-get install -y libhdf5-dev - - - name: Setup Go environment - if: ${{ inputs.require_go == 'true' }} - uses: ./.github/actions/setup-go - - - name: Setup Helm environment - if: ${{ inputs.require_helm == 'true' }} - uses: ./.github/actions/setup-helm - - - name: Wait for target Docker images - if: startsWith( github.ref, 'refs/tags/') - uses: ./.github/actions/wait-for-docker-image - with: - images: ${{ inputs.target_images }} - - - name: Determine Docker image tag - id: determine_tag_name - uses: ./.github/actions/determine-docker-image-tag - - - name: Specify container versions - id: specify_container_versions - uses: ./.github/actions/detect-docker-image-tags - with: - tag_name: ${{ steps.determine_tag_name.outputs.PRIMARY_TAG }} - images: ${{ inputs.target_images }} - - - uses: ./.github/actions/setup-k3d - if: ${{ inputs.require_k3d == 'true' }} - with: - agents: 3 - ingress_port: ${{ inputs.ingress_port }} - options: "--image docker.io/rancher/k3s:latest" - - - name: Check Kubernetes cluster - shell: bash - run: | - kubectl cluster-info diff --git a/.github/actions/setup-go/action.yaml b/.github/actions/setup-go/action.yaml deleted file mode 100644 index 7ac61ed210..0000000000 --- a/.github/actions/setup-go/action.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: "Setup Go environment" -description: "A action to set up Go environment" - -inputs: - go_version: - description: "The Go version to use. The default version is `versions/GO_VERSION`" - required: false - default: "" - -runs: - using: "composite" - steps: - - name: Fetch Go version - id: go_version - shell: bash - run: | - GO_VERSION=${GO_VERSION:-`make version/go`} - - echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT - env: - GO_VERSION: ${{ inputs.go_version }} - - - uses: actions/setup-go@v3 - with: - go-version: ${{ steps.go_version.outputs.version }} - - - name: Check Go version - shell: bash - run: | - go version diff --git a/.github/actions/setup-helm/action.yaml b/.github/actions/setup-helm/action.yaml deleted file mode 100644 index 9c2d7f3111..0000000000 --- a/.github/actions/setup-helm/action.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: "Setup Helm environment" -description: "A action to set up Helm environment" - -inputs: - helm_version: - description: "The Helm version to use. The default version is `versions/HELM_VERSION`" - required: false - default: "" - -runs: - using: "composite" - steps: - - name: Fetch Helm version - id: helm_version - shell: bash - run: | - HELM_VERSION=${HELM_VERSION:-`make version/helm`} - - echo "version=${HELM_VERSION}" >> $GITHUB_OUTPUT - env: - HELM_VERSION: ${{ inputs.helm_version }} - - - uses: azure/setup-helm@v3 - with: - version: ${{ steps.helm_version.outputs.version }} - - - name: Check Helm version - shell: bash - run: | - helm version diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml deleted file mode 100644 index b23ef0c7f9..0000000000 --- a/.github/actions/setup-k3d/action.yaml +++ /dev/null @@ -1,71 +0,0 @@ -name: "Setup k3d environment" -description: "A action to set up k3d (k3s in Docker)" - -inputs: - version: - description: "k3d version" - required: false - default: "latest" - name: - description: "Cluster name" - required: false - default: "vald" - ingress_port: - description: 'If it is not "0", ingress will be exposed to the specified port' - required: false - default: "0" - agents: - description: "Number of agents" - required: false - default: "3" - options: - description: "Options for k3d cluster create command" - required: false - default: "" - -runs: - using: "composite" - steps: - - name: Detect k3d version - id: k3d_version - shell: bash - run: | - if [ "${K3D_VERSION}" != "latest" ]; then - TAG="TAG=v${K3D_VERSION}" - fi - echo "tag=${TAG}" >> $GITHUB_OUTPUT - env: - K3D_VERSION: ${{ inputs.version }} - - - name: Install k3d - shell: bash - run: | - curl -s ${REPO_URL} | ${{ steps.k3d_version.outputs.tag }} bash - env: - REPO_URL: "https://github.com/raw/k3d-io/k3d/main/install.sh" - - - name: Check k3d version - shell: bash - run: | - k3d version - - - name: Setup k3d options - shell: bash - id: k3d_options - run: | - if [ "${AGENTS}" != 0 ]; then - OPTIONS="${OPTIONS} --agents ${AGENTS}" - fi - if [ "${INGRESS_PORT}" != 0 ]; then - OPTIONS="${OPTIONS} -p ${INGRESS_PORT}:80@loadbalancer" - fi - echo "options=${OPTIONS}" >> $GITHUB_OUTPUT - env: - AGENTS: ${{ inputs.agents }} - INGRESS_PORT: ${{ inputs.ingress_port }} - OPTIONS: ${{ inputs.options }} - - - name: Create k8s cluster - shell: bash - run: | - k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }} diff --git a/.github/actions/setup-yq/action.yaml b/.github/actions/setup-yq/action.yaml deleted file mode 100644 index 13ad2a1614..0000000000 --- a/.github/actions/setup-yq/action.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: "Setup yq environment" -description: "A action to set up yq" - -inputs: - yq_version: - description: "The yq version to use. The default version is `versions/YQ_VERSION`" - required: false - default: "" - -runs: - using: "composite" - steps: - - name: Fetch yq version - id: yq_version - shell: bash - run: | - YQ_VERSION=${YQ_VERSION:-`make version/yq`} - - echo "version=${YQ_VERSION}" >> $GITHUB_OUTPUT - env: - YQ_VERSION: ${{ inputs.yq_version }} - - - name: Install yq - shell: bash - run: | - curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 - chmod a+x /usr/local/bin/yq - env: - YQ_VERSION: ${{ steps.yq_version.outputs.version }} - - - name: Check yq version - shell: bash - run: | - yq --version diff --git a/.github/actions/wait-for-docker-image/action.yaml b/.github/actions/wait-for-docker-image/action.yaml index 851041813a..375e03caae 100644 --- a/.github/actions/wait-for-docker-image/action.yaml +++ b/.github/actions/wait-for-docker-image/action.yaml @@ -14,8 +14,7 @@ # limitations under the License. # name: "Wait for Docker Images" -description: "A action to wait for Docker images to be published" - +description: "Wait for Docker images to be published" inputs: images: description: "image names" @@ -25,7 +24,6 @@ inputs: vdaas/vald-lb-gateway \ vdaas/vald-manager-index" outputs: {} - runs: using: "composite" steps: diff --git a/.github/chatops_commands.md b/.github/chatops_commands.md index c08eb41a2d..b06976e1ae 100644 --- a/.github/chatops_commands.md +++ b/.github/chatops_commands.md @@ -1,5 +1,7 @@ - :ok_woman: `/approve` - approve +- :love_letter: `/changelog` - replace the PR body by changelog details - :bento: `/format` - format codes and add licenses - :white_check_mark: `/gen-test` - generate test codes - :label: `/label` - add labels +- :rewind: `/rebase` - rebase main - :end: :two: :end: `/label actions/e2e-deploy` - run E2E deploy & integration test diff --git a/.github/helm/values/values-chaos.yaml b/.github/helm/values/values-chaos.yaml index 853c82faf0..8ca30303ce 100644 --- a/.github/helm/values/values-chaos.yaml +++ b/.github/helm/values/values-chaos.yaml @@ -17,7 +17,6 @@ defaults: grpc: client: - health_check_duration: 50ms dial_option: enable_backoff: true server_config: @@ -34,13 +33,6 @@ defaults: gateway: lb: enabled: true - ingress: - enabled: true - host: "localhost" - service: - # NOTE: https://doc.traefik.io/traefik/routing/providers/kubernetes-ingress/#on-service - annotations: - traefik.ingress.kubernetes.io/service.serversscheme: h2c minReplicas: 2 hpa: enabled: false @@ -49,19 +41,12 @@ gateway: cpu: 100m memory: 50Mi gateway_config: - index_replica: 3 - discoverer: - duration: 50ms + index_replica: 2 agent: minReplicas: 5 maxReplicas: 10 podManagementPolicy: Parallel - persistentVolume: - enabled: true - storageClass: local-path - accessMode: ReadWriteOnce - size: 200Mi hpa: enabled: false resources: @@ -69,8 +54,6 @@ agent: cpu: 100m memory: 50Mi ngt: - index_path: "/var/ngt/index" - enable_in_memory_mode: false auto_index_duration_limit: 3m auto_index_check_duration: 1m auto_index_length: 1000 @@ -85,7 +68,7 @@ discoverer: cpu: 100m memory: 50Mi discoverer: - discovery_duration: 50ms + discovery_duration: 300ms manager: index: diff --git a/.github/valdrelease/valdrelease.yaml b/.github/valdrelease/valdrelease.yaml index 94274292b4..47f120110b 100644 --- a/.github/valdrelease/valdrelease.yaml +++ b/.github/valdrelease/valdrelease.yaml @@ -20,8 +20,6 @@ metadata: name: vald-cluster spec: defaults: - image: - tag: nightly logging: level: info networkPolicy: diff --git a/.github/workflows/_docker-image-scan.yaml b/.github/workflows/_docker-image-scan.yaml deleted file mode 100644 index 1e20a00a42..0000000000 --- a/.github/workflows/_docker-image-scan.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: "Docker image scanning" -on: - workflow_call: - inputs: - target: - type: string - description: 'Build target. e.g. `agent-ngt`' - required: true - -jobs: - scan: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Build the Docker image - id: build_image - run: | - make docker/build/${TARGET} - imagename=`make docker/name/${TARGET}` - docker tag ${imagename} ${imagename}:${{ github.sha }} - echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT - env: - DOCKER_BUILDKIT: 1 - TARGET: ${{ inputs.target }} - - - name: Scan the Docker image - uses: ./.github/actions/scan-docker-image - with: - image_ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" - severity: "HIGH,CRITICAL" diff --git a/.github/workflows/_docker-image.yaml b/.github/workflows/_docker-image.yaml deleted file mode 100644 index 1017841996..0000000000 --- a/.github/workflows/_docker-image.yaml +++ /dev/null @@ -1,94 +0,0 @@ -name: "Build docker image" -on: - workflow_call: - inputs: - target: - type: string - description: "Build target. e.g. `agent-ngt`" - required: true - platforms: - type: string - description: "If it is specified, specified platforms will be used." - required: false - default: "" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }}-${{ inputs.target }} - cancel-in-progress: true - -jobs: - dump-contexts-to-log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - - build: - runs-on: ubuntu-latest - if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} - steps: - - name: Get ref - id: ref - run: | - if [ ${{ github.event.pull_request.head.sha }} != "" ]; then - echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT - else - echo ref=${{ github.sha }} >> $GITHUB_OUTPUT - fi - - - uses: actions/checkout@v3 - with: - ref: ${{ steps.ref.outputs.ref }} - - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup QEMU - uses: docker/setup-qemu-action@v2 - with: - platforms: all - - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - buildkitd-flags: "--debug" - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASS }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ secrets.PACKAGE_USER }} - password: ${{ secrets.PACKAGE_TOKEN }} - - - name: Build and Publish - id: build_and_publish - uses: ./.github/actions/docker-build - with: - target: ${{ inputs.target }} - platforms: ${{ inputs.platforms }} - builder: ${{ steps.buildx.outputs.name }} - - - name: Scan the Docker image - if: startsWith( github.ref, 'refs/tags/') - uses: ./.github/actions/scan-docker-image - with: - image_ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" - - slack: - runs-on: ubuntu-latest - needs: [build] - if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/notify-slack - with: - author_name: ${{ inputs.target }} image build - slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/_update-protobuf.yaml b/.github/workflows/_update-protobuf.yaml deleted file mode 100644 index b6b28a0fa8..0000000000 --- a/.github/workflows/_update-protobuf.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (C) 2019-2023 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Update protobuf" -on: - workflow_call: - inputs: - repo: - type: string - description: "Dispatch target repository" - required: true - default: "" - -jobs: - dispatch: - runs-on: ubuntu-latest - steps: - - name: Dispatch - run: | - curl --fail -u "${USER}:${TOKEN}" -X POST https://github.com/gitapi/repos/vdaas/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' - env: - REPO: ${{ inputs.repo }} - USER: ${{ secrets.DISPATCH_USER }} - TOKEN: ${{ secrets.DISPATCH_TOKEN }} - diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index dab4837d72..2f68eb5ba0 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -20,12 +20,34 @@ on: - created jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build-linux: runs-on: ubuntu-latest container: @@ -34,33 +56,64 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 10 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - uses: ./.github/actions/setup-go - + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} - name: Build and zip run: | make binary/build/zip - - name: Upload artifact uses: actions/upload-artifact@v3 with: name: artifacts-linux path: ./artifacts/ - + # build-macos: ## or using cross-compiler? + # runs-on: macos-latest + # steps: + # - uses: actions/checkout@v3 + # with: + # fetch-depth: 10 + # - name: Fetch golang version + # run: | + # GO_VERSION=`make version/go` + # echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + # id: golang_version + # - uses: actions/setup-go@v3 + # with: + # go-version: ${{ steps.golang_version.outputs.version }} + # - name: Build and zip + # run: | + # export PATH=$PATH:$(go env GOPATH)/bin + # brew install llvm libomp protobuf ngt + # make CXXFLAGS="-I/usr/local/opt/llvm/include -mno-avx512f -mno-avx512dq -mno-avx512cd -mno-avx512bw -mno-avx512vl" binary/build/zip + # - name: Upload artifact + # uses: actions/upload-artifact@v3 + # with: + # name: artifacts-macos + # path: ./artifacts publish: runs-on: ubuntu-latest needs: - build-linux + # - build-macos + # - build-windows steps: - uses: actions/download-artifact@v2 with: name: artifacts-linux path: tmp/linux - + # - uses: actions/download-artifact@v2 + # with: + # name: artifacts-macos + # path: tmp/macos - uses: shogo82148/actions-upload-release-asset@v1 with: upload_url: ${{ github.event.release.upload_url }} diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml index 6aab18aa03..ced2f7cdf2 100644 --- a/.github/workflows/build-protobuf.yml +++ b/.github/workflows/build-protobuf.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Build protobuf" +name: build protobuf on: push: branches: @@ -27,23 +27,46 @@ on: - "versions/GO_VERSION" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: + name: build all runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - - - name: Set Git config + with: + persist-credentials: false + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Build protobuf + - name: build protobuf run: | make proto/all diff --git a/.github/workflows/chatops-help.yml b/.github/workflows/chatops-help.yml index 7ad564d8be..42478c5fbb 100644 --- a/.github/workflows/chatops-help.yml +++ b/.github/workflows/chatops-help.yml @@ -17,18 +17,40 @@ name: "ChatOps help" on: pull_request: types: [opened] - jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - - print: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} + print-help: + name: print help runs-on: ubuntu-latest steps: - - name: Comment to PR + - name: comment to PR run: | curl -o chatops_commands.md https://github.com/raw/vdaas/vald/main/.github/chatops_commands.md BODY=`cat chatops_commands.md | sed ':a;N;$!ba;s/\n/\\\\n/g'` diff --git a/.github/workflows/chatops.yml b/.github/workflows/chatops.yml index d3ca87270e..d3c77552ed 100644 --- a/.github/workflows/chatops.yml +++ b/.github/workflows/chatops.yml @@ -13,35 +13,59 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: ChatOps on: issue_comment: types: [created] - +env: + YQ_VERSION: 3.2.1 +name: ChatOps jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} label: name: Add labels runs-on: ubuntu-latest if: github.event.comment.user.login != 'vdaas-ci' steps: - - name: Check PR Comments + - name: check PR Comments id: check_comments_label uses: machine-learning-apps/actions-chatops@master with: TRIGGER_PHRASE: "/label" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./.github/actions/setup-yq + - name: install yq if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' - - - name: Check permissions + run: | + sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + sudo chmod a+x /usr/local/bin/yq + - name: check permissions if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' id: check_permissions run: | @@ -61,8 +85,7 @@ jobs: env: USERNAME: ${{ steps.check_comments_label.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: label - - - name: Add label + - name: add label if: steps.check_comments_label.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | labels=`echo "${TRAILING_LINE}" | jq -cMR 'split(" ")'` @@ -78,23 +101,96 @@ jobs: ISSUE_URL: ${{ github.event.issue.labels_url }} USERNAME: ${{ steps.check_comments_label.outputs.COMMENTER_USERNAME }} TRAILING_LINE: ${{ steps.check_comments_label.outputs.TRAILING_LINE }} + changelog: + name: Add draft comment for a new CHANGELOG entry + runs-on: ubuntu-latest + if: github.event.comment.user.login != 'vdaas-ci' + steps: + - name: check PR Comments + id: check_comments_changelog + uses: machine-learning-apps/actions-chatops@master + with: + TRIGGER_PHRASE: "/changelog" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: install yq + if: steps.check_comments_changelog.outputs.BOOL_TRIGGERED == 'true' + run: | + sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + sudo chmod a+x /usr/local/bin/yq + - name: check permissions + if: steps.check_comments_changelog.outputs.BOOL_TRIGGERED == 'true' + id: check_permissions + run: | + curl -o chatops_permissions.yaml https://github.com/raw/vdaas/vald/main/.github/chatops_permissions.yaml + roles=`yq r chatops_permissions.yaml "users.${USERNAME}.role.*"` + if [ "$roles" = "" ]; then + roles=`yq r chatops_permissions.yaml "default-roles.*"` + fi + for role in `echo $roles` + do + if yq r chatops_permissions.yaml "roles.${role}.policies.*" | grep "^${REQUIRED_POLICY}$" > /dev/null 2>&1 ; then + echo "[OK] changelog requested by ${USERNAME}" + echo "EXECUTABLE=true" >> $GITHUB_OUTPUT + break + fi + done + env: + USERNAME: ${{ steps.check_comments_changelog.outputs.COMMENTER_USERNAME }} + REQUIRED_POLICY: changelog + - uses: actions/checkout@v3 + if: steps.check_comments_changelog.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' + with: + fetch-depth: 0 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: comment to PR + if: steps.check_comments_changelog.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' + run: | + git checkout main + PREVIOUS_VERSION=$(git describe --tags $(git rev-list --tags --max-count=1)) + BODY=`git log --pretty=format:'- %s' ${PREVIOUS_VERSION}..main | grep "#[[:digit:]]\+" | sed -e "s/\[\(patch\|minor\|major\)\] *//g" | sed -e "s%#\([[:digit:]]\+\)%[&](https://github.com/vdaas/vald/pull/\1)%g" | sed ':a;N;$!ba;s/\n/\\\\n/g'` + CURRENT="- ${PR_TITLE} ([#${PR_NUM}](https://github.com/vdaas/vald/pull/${PR_NUM}))" + + curl --include --verbose --fail \ + -H "Accept: application/json" \ + -H "Content-Type:application/json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + --request PATCH \ + --data "{\"body\": \"${CURRENT}\\n${BODY}\"}" \ + https://github.com/gitapi/repos/vdaas/vald/pulls/${PR_NUM} + + curl --include --verbose --fail \ + -H "Accept: application/json" \ + -H "Content-Type:application/json" \ + -H "Authorization: token ${GITHUB_TOKEN}" \ + --request POST \ + --data "{\"body\": \"**[CHANGELOG]** The description of this PR is updated. Please edit it.\"}" \ + $API_URL + env: + GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} + API_URL: ${{ github.event.issue.comments_url }} + PR_TITLE: ${{ github.event.issue.title }} + PR_NUM: ${{ github.event.issue.number }} rebase: name: Rebase runs-on: ubuntu-latest if: github.event.comment.user.login != 'vdaas-ci' steps: - - name: Check PR Comments + - name: check PR Comments id: check_comments_rebase uses: machine-learning-apps/actions-chatops@master with: TRIGGER_PHRASE: "/rebase" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./.github/actions/setup-yq + - name: install yq if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' - - - name: Check permissions + run: | + sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + sudo chmod a+x /usr/local/bin/yq + - name: check permissions if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' id: check_permissions run: | @@ -114,8 +210,7 @@ jobs: env: USERNAME: ${{ steps.check_comments_rebase.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: rebase - - - name: Comment back to PR + - name: comment back to PR if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | curl --include --verbose --fail \ @@ -130,24 +225,20 @@ jobs: API_URL: ${{ github.event.issue.comments_url }} BRANCH: ${{ steps.check_comments_rebase.outputs.BRANCH_NAME }} USERNAME: ${{ steps.check_comments_rebase.outputs.COMMENTER_USERNAME }} - - uses: actions/checkout@v3 if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Automatic Rebase if: steps.check_comments_rebase.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' uses: cirrus-actions/rebase@1.5 env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - - - name: Failure comment + - name: failure comment if: failure() run: | curl --include --verbose --fail \ @@ -160,7 +251,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: ${{ github.event.issue.comments_url }} - gentest: name: Add tests needs: @@ -175,10 +265,11 @@ jobs: TRIGGER_PHRASE: "/gen-test" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./.github/actions/setup-yq + - name: install yq if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' - + run: | + sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + sudo chmod a+x /usr/local/bin/yq - name: check permissions if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' id: check_permissions @@ -199,7 +290,6 @@ jobs: env: USERNAME: ${{ steps.check_comments_gen_test.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: gen-test - - name: check executable if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | @@ -215,26 +305,29 @@ jobs: USERNAME: ${{ steps.check_comments_gen_test.outputs.COMMENTER_USERNAME }} PR_AUTHOR: ${{ github.event.issue.user.login }} API_URL: ${{ github.event.issue.comments_url }} - - uses: actions/checkout@v3 if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: fetch-depth: 0 - - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - uses: crazy-max/ghaction-import-gpg@v4 if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - - uses: ./.github/actions/setup-go + - name: Fetch golang version if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' + with: + go-version: ${{ steps.golang_version.outputs.version }} - name: Generate tests and push id: gen_test if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' @@ -269,7 +362,6 @@ jobs: PR_INFO_URL: ${{ github.event.issue.pull_request.url }} PR_AUTHOR: ${{ github.event.issue.user.login }} PR_NUM: ${{ github.event.issue.number }} - - name: failure comment if: failure() run: | @@ -284,7 +376,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: ${{ github.event.issue.comments_url }} - format: name: Add license and run golines + gofumpt + goimports needs: @@ -300,10 +391,11 @@ jobs: TRIGGER_PHRASE: "/format" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./.github/actions/setup-yq + - name: install yq if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' - + run: | + sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + sudo chmod a+x /usr/local/bin/yq - name: check permissions if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' id: check_permissions @@ -324,7 +416,6 @@ jobs: env: USERNAME: ${{ steps.check_comments_format.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: format - - name: check executable if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | @@ -339,27 +430,30 @@ jobs: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} USERNAME: ${{ steps.check_comments_format.outputs.COMMENTER_USERNAME }} API_URL: ${{ github.event.issue.comments_url }} - - uses: actions/checkout@v3 if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} - - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - uses: crazy-max/ghaction-import-gpg@v4 if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - - uses: ./.github/actions/setup-go - if: steps.check_comments_gen_test.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' - + - name: Fetch golang version + if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' + with: + go-version: ${{ steps.golang_version.outputs.version }} - name: update and push id: format_push if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' @@ -401,7 +495,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} PR_INFO_URL: ${{ github.event.issue.pull_request.url }} PR_NUM: ${{ github.event.issue.number }} - - name: no changes if: steps.check_comments_format.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' && steps.format_push.outputs.UPDATED == 'false' run: | @@ -415,7 +508,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: ${{ github.event.issue.comments_url }} - - name: failure comment if: failure() run: | @@ -429,7 +521,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: ${{ github.event.issue.comments_url }} - approve: name: Approve needs: @@ -445,10 +536,11 @@ jobs: TRIGGER_PHRASE: "/approve" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./.github/actions/setup-yq + - name: install yq if: steps.check_comments_approve.outputs.BOOL_TRIGGERED == 'true' - + run: | + sudo curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 + sudo chmod a+x /usr/local/bin/yq - name: check permissions if: steps.check_comments_approve.outputs.BOOL_TRIGGERED == 'true' id: check_permissions @@ -469,7 +561,6 @@ jobs: env: USERNAME: ${{ steps.check_comments_approve.outputs.COMMENTER_USERNAME }} REQUIRED_POLICY: approve - - name: approve if: steps.check_comments_approve.outputs.BOOL_TRIGGERED == 'true' && steps.check_permissions.outputs.EXECUTABLE == 'true' run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9904976f29..e95b13890c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Code scanning" +name: "Code scanning - action" + on: push: branches: @@ -33,17 +34,36 @@ on: - cron: "0 1 * * *" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - - codeql-build: - name: CodeQL + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} + CodeQL-Build: runs-on: ubuntu-latest - container: - image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - name: Checkout repository uses: actions/checkout@v3 @@ -51,19 +71,20 @@ jobs: # We must fetch at least the immediate parents so that if this is # a pull request then we can checkout the head. fetch-depth: 2 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - + - name: Install dependencies + run: | + sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends libhdf5-dev + sudo make ngt/install - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: languages: go config-file: ./.github/codeql/codeql-config.yaml - - name: Autobuild uses: github/codeql-action/autobuild@v2 - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 7f007ff8bf..78c7abeace 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Coverage reporting" +name: "Coverage" on: push: branches: @@ -31,36 +31,56 @@ on: - "cmd/**" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} coverage: + name: Coverage runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 + - name: Check out code. + uses: actions/checkout@v3 with: fetch-depth: 10 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Run coverage continue-on-error: true run: | make coverage - - name: Upload coverage report to Codecov uses: codecov/codecov-action@v3 with: token: ${{secrets.CODECOV_TOKEN}} file: ./coverage.out - - name: Upload coverage report to deepsource run: | mv ./coverage.out ./cover.out diff --git a/.github/workflows/detect-internal-config-changes.yml b/.github/workflows/detect-internal-config-changes.yml index 935268ee70..4ff5904d7a 100644 --- a/.github/workflows/detect-internal-config-changes.yml +++ b/.github/workflows/detect-internal-config-changes.yml @@ -21,13 +21,36 @@ on: - "!internal/config/**/*_test.go" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} warning: + name: warn internal config changes runs-on: ubuntu-latest steps: - name: Warn diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml index 568ff361e6..188ad2b517 100644 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ b/.github/workflows/dockers-agent-ngt-image.yml @@ -25,7 +25,6 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-ngt-image.yml" - "go.mod" - "go.sum" @@ -42,7 +41,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-ngt-image.yml" - "go.mod" - "go.sum" @@ -59,7 +57,6 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-ngt-image.yml" - "go.mod" - "go.sum" @@ -74,9 +71,121 @@ on: - "versions/GO_VERSION" - "versions/NGT_VERSION" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: agent-ngt - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: agent-ngt + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: agent-ngt image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-agent-sidecar-image.yml b/.github/workflows/dockers-agent-sidecar-image.yml index 2bf26f8c29..f067deb575 100644 --- a/.github/workflows/dockers-agent-sidecar-image.yml +++ b/.github/workflows/dockers-agent-sidecar-image.yml @@ -25,7 +25,6 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-sidecar-image.yml" - "go.mod" - "go.sum" @@ -42,7 +41,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-sidecar-image.yml" - "go.mod" - "go.sum" @@ -59,7 +57,6 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-agent-sidecar-image.yml" - "go.mod" - "go.sum" @@ -74,9 +71,121 @@ on: - "dockers/agent/sidecar/Dockerfile" - "versions/GO_VERSION" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: agent-sidecar - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: agent-sidecar + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: agent-sidecar image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-ci-container-image.yml b/.github/workflows/dockers-ci-container-image.yml index baf5c21378..d6e6636ba1 100644 --- a/.github/workflows/dockers-ci-container-image.yml +++ b/.github/workflows/dockers-ci-container-image.yml @@ -20,7 +20,6 @@ on: - main paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-ci-container-image.yml" - "dockers/ci/**" - "Makefile" @@ -30,7 +29,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-ci-container-image.yml" - "dockers/ci/**" - "Makefile" @@ -40,20 +38,90 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-ci-container-image.yml" - "dockers/ci/**" - "Makefile" - "Makefile.d/**" - "versions/GO_VERSION" - "versions/NGT_VERSION" - schedule: - - cron: "0 1 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: ci-container - platforms: linux/amd64 - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: ci-container + builder: ${{ steps.buildx.outputs.name }} + platforms: linux/amd64 diff --git a/.github/workflows/dockers-dev-container-image.yml b/.github/workflows/dockers-dev-container-image.yml index 7994d1f7d8..a3ec496967 100644 --- a/.github/workflows/dockers-dev-container-image.yml +++ b/.github/workflows/dockers-dev-container-image.yml @@ -20,31 +20,101 @@ on: - main paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - "dockers/ci/**" - "dockers/dev/**" pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - "dockers/ci/**" - "dockers/dev/**" pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-dev-container-image.yml" - "dockers/ci/**" - "dockers/dev/**" schedule: - cron: "0 1 * * *" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: dev-container - platforms: linux/amd64 - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: dev-container + builder: ${{ steps.buildx.outputs.name }} + platforms: linux/amd64 diff --git a/.github/workflows/dockers-discoverer-k8s-image.yml b/.github/workflows/dockers-discoverer-k8s-image.yml index d0d12c260d..767c1b6a6c 100644 --- a/.github/workflows/dockers-discoverer-k8s-image.yml +++ b/.github/workflows/dockers-discoverer-k8s-image.yml @@ -25,7 +25,6 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-discoverer-k8s-image.yml" - "go.mod" - "go.sum" @@ -40,7 +39,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-discoverer-k8s-image.yml" - "go.mod" - "go.sum" @@ -55,7 +53,6 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-discoverer-k8s-image.yml" - "go.mod" - "go.sum" @@ -68,9 +65,121 @@ on: - "dockers/discoverer/k8s/Dockerfile" - "versions/GO_VERSION" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: discoverer-k8s - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: discoverer-k8s + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: discoverer-k8s image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-gateway-filter-image.yml b/.github/workflows/dockers-gateway-filter-image.yml index a8a1f56027..58e2a91628 100644 --- a/.github/workflows/dockers-gateway-filter-image.yml +++ b/.github/workflows/dockers-gateway-filter-image.yml @@ -25,7 +25,6 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-filter-image.yml" - "go.mod" - "go.sum" @@ -43,7 +42,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-filter-image.yml" - "go.mod" - "go.sum" @@ -61,7 +59,6 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-filter-image.yml" - "go.mod" - "go.sum" @@ -77,9 +74,128 @@ on: - "dockers/gateway/filter/Dockerfile" - "versions/GO_VERSION" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: gateway-filter - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-vald-gateway-filter-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-vald-gateway-filter- + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: gateway-filter + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: vald-filter-gateway image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-gateway-lb-image.yml b/.github/workflows/dockers-gateway-lb-image.yml index 69e80cb921..4a4b7c0a7c 100644 --- a/.github/workflows/dockers-gateway-lb-image.yml +++ b/.github/workflows/dockers-gateway-lb-image.yml @@ -25,7 +25,6 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-lb-image.yml" - "go.mod" - "go.sum" @@ -43,7 +42,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-lb-image.yml" - "go.mod" - "go.sum" @@ -61,7 +59,6 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-gateway-lb-image.yml" - "go.mod" - "go.sum" @@ -77,9 +74,121 @@ on: - "dockers/gateway/lb/Dockerfile" - "versions/GO_VERSION" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: gateway-lb - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: gateway-lb + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: vald-lb-gateway image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-helm-operator-image.yml b/.github/workflows/dockers-helm-operator-image.yml index 48b68fca3f..6593b02653 100644 --- a/.github/workflows/dockers-helm-operator-image.yml +++ b/.github/workflows/dockers-helm-operator-image.yml @@ -25,7 +25,6 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-helm-operator-image.yml" - "dockers/operator/helm/Dockerfile" - "charts/vald/Chart.yaml" @@ -38,7 +37,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-helm-operator-image.yml" - "dockers/operator/helm/Dockerfile" - "charts/vald/Chart.yaml" @@ -51,7 +49,6 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-helm-operator-image.yml" - "dockers/operator/helm/Dockerfile" - "charts/vald/Chart.yaml" @@ -62,9 +59,121 @@ on: - "charts/vald-helm-operator/templates/**" - "versions/OPERATOR_SDK_VERSION" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: operator/helm - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: operator/helm + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: helm-operator image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-image-scan.yml b/.github/workflows/dockers-image-scan.yml index 61835d6a5c..f6dd2c1581 100644 --- a/.github/workflows/dockers-image-scan.yml +++ b/.github/workflows/dockers-image-scan.yml @@ -23,48 +23,295 @@ concurrency: cancel-in-progress: true jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} agent-ngt: - uses: ./.github/workflows/_docker-image-scan.yaml - with: - target: agent-ngt - + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/agent-ngt + imagename=`make docker/name/agent-ngt` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" agent-sidecar: - uses: ./.github/workflows/_docker-image-scan.yaml - with: - target: agent-sidecar - + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/agent-sidecar + imagename=`make docker/name/agent-sidecar` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" discoverer-k8s: - uses: ./.github/workflows/_docker-image-scan.yaml - with: - target: discoverer-k8s - + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/discoverer-k8s + imagename=`make docker/name/discoverer-k8s` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" manager-index: - uses: ./.github/workflows/_docker-image-scan.yaml - with: - target: manager-index - + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/manager-index + imagename=`make docker/name/manager-index` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" operator-helm: - uses: ./.github/workflows/_docker-image-scan.yaml - with: - target: operator/helm - + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/operator/helm + imagename=`make docker/name/operator/helm` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" loadtest: - uses: ./.github/workflows/_docker-image-scan.yaml - with: - target: loadtest - + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/loadtest + imagename=`make docker/name/loadtest` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" gateway-lb: - uses: ./.github/workflows/_docker-image-scan.yaml - with: - target: gateway-lb - + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/gateway-lb + imagename=`make docker/name/gateway-lb` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" gateway-filter: - uses: ./.github/workflows/_docker-image-scan.yaml - with: - target: gateway-filter + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Build the Docker image + id: build_image + run: | + make docker/build/gateway-filter + imagename=`make docker/name/gateway-filter` + docker tag ${imagename} ${imagename}:${{ github.sha }} + echo "IMAGE_NAME=${imagename}" >> $GITHUB_OUTPUT + env: + DOCKER_BUILDKIT: 1 + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "table" + - name: Run vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_image.outputs.IMAGE_NAME }}:${{ github.sha }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + severity: "HIGH,CRITICAL" + - name: Upload Trivy scan results to Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" diff --git a/.github/workflows/dockers-loadtest-image.yml b/.github/workflows/dockers-loadtest-image.yml index 21a8df8ba5..e87654247c 100644 --- a/.github/workflows/dockers-loadtest-image.yml +++ b/.github/workflows/dockers-loadtest-image.yml @@ -25,7 +25,6 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-loadtest-image.yml" - "go.mod" - "go.sum" @@ -41,7 +40,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-loadtest-image.yml" - "go.mod" - "go.sum" @@ -57,7 +55,6 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-loadtest-image.yml" - "go.mod" - "go.sum" @@ -71,10 +68,122 @@ on: - "dockers/tools/cli/loadtest/Dockerfile" - "versions/GO_VERSION" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: loadtest - platforms: linux/amd64 - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: loadtest + builder: ${{ steps.buildx.outputs.name }} + platforms: linux/amd64 + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: vald-loadtest image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/dockers-manager-index-image.yml b/.github/workflows/dockers-manager-index-image.yml index b517d46a24..f20f561434 100644 --- a/.github/workflows/dockers-manager-index-image.yml +++ b/.github/workflows/dockers-manager-index-image.yml @@ -25,7 +25,6 @@ on: - "v*.*.*-*" paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-manager-index-image.yml" - "go.mod" - "go.sum" @@ -41,7 +40,6 @@ on: pull_request: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-manager-index-image.yml" - "go.mod" - "go.sum" @@ -57,7 +55,6 @@ on: pull_request_target: paths: - ".github/actions/docker-build/actions.yaml" - - ".github/workflows/_docker-image.yaml" - ".github/workflows/dockers-manager-index-image.yml" - "go.mod" - "go.sum" @@ -71,9 +68,121 @@ on: - "dockers/manager/index/Dockerfile" - "versions/GO_VERSION" +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }} + cancel-in-progress: true + jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: manager-index - secrets: inherit + strategy: + max-parallel: 4 + runs-on: ubuntu-latest + if: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || (github.event.pull_request.head.repo.fork == true && github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'ci/approved')) || (github.event_name == 'push' && github.ref == 'refs/heads/main') || startsWith( github.ref, 'refs/tags/') }} + steps: + - name: Get ref + id: ref + run: | + if [ ${{ github.event.pull_request.head.sha }} != "" ]; then + echo ref=${{ github.event.pull_request.head.sha }} >> $GITHUB_OUTPUT + else + echo ref=${{ github.sha }} >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + with: + ref: ${{ steps.ref.outputs.ref }} + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Setup QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: all + - name: Setup Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + buildkitd-flags: "--debug" + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PASS }} + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.PACKAGE_USER }} + password: ${{ secrets.PACKAGE_TOKEN }} + - name: Build and Publish + id: build_and_publish + uses: ./.github/actions/docker-build + with: + target: manager-index + builder: ${{ steps.buildx.outputs.name }} + - name: Initialize CodeQL + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/init@v2 + - name: Run vulnerability scanner (table) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "table" + - name: Run vulnerability scanner (sarif) + if: startsWith( github.ref, 'refs/tags/') + uses: aquasecurity/trivy-action@master + with: + image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}" + format: "template" + template: "@/contrib/sarif.tpl" + output: "trivy-results.sarif" + - name: Upload Trivy scan results to Security tab + if: startsWith( github.ref, 'refs/tags/') + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-results.sarif" + slack: + name: Slack notification + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: manager-index image build + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml index ebee729e56..d9cf6b5839 100644 --- a/.github/workflows/e2e-chaos.yaml +++ b/.github/workflows/e2e-chaos.yaml @@ -25,207 +25,407 @@ on: types: - "labeled" -env: - VALUES: .github/helm/values/values-chaos.yaml - DATASET: fashion-mnist-784-euclidean.hdf5 - INGRESS_PORT: 8081 - jobs: - dump-contexts-to-log: - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} agent-failure: name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)" - needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags with: - ingress_port: ${{ env.INGRESS_PORT }} - - - name: Deploy Vald + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + options: "--image docker.io/rancher/k3s:latest" + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: ${{ env.VALUES }} + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-chaos.yaml wait_for_selector: app=vald-lb-gateway - - - name: Deploy Chaos Mesh - uses: ./.github/actions/deploy-chaos-mesh + - name: deploy Chaos Mesh + run: | + make kubectl/install + curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s + helm install \ + --set podChaos.failure.enabled=true \ + vald-chaos-test tests/chaos/chart + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 with: - helm_extra_options: "--set podChaos.failure.enabled=true" - - - name: Run Insert and Search operations + go-version: ${{ steps.golang_version.outputs.version }} + - name: run Insert and Search jobs run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_HOST=localhost \ - E2E_BIND_PORT=8081 \ - E2E_PORTFORWARD_ENABLED=false \ - E2E_DATASET_NAME=${{ env.DATASET }} \ + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=5000 \ - E2E_SEARCH_COUNT=5000 \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ e2e/insert/search - + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} random-pod-failure: name: "E2E chaos test (random Pod failure: to test redundancy)" - needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags with: - ingress_port: ${{ env.INGRESS_PORT }} - - - name: Deploy Vald + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + options: "--image docker.io/rancher/k3s:latest" + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: ${{ env.VALUES }} + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-chaos.yaml wait_for_selector: app=vald-lb-gateway - - - name: Deploy Chaos Mesh - uses: ./.github/actions/deploy-chaos-mesh + - name: deploy Chaos Mesh + run: | + make kubectl/install + curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s + helm install \ + --set podChaos.kill.enabled=true \ + vald-chaos-test tests/chaos/chart + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 with: - helm_extra_options: "--set podChaos.kill.enabled=true" - - - name: Run Insert and Search operations + go-version: ${{ steps.golang_version.outputs.version }} + - name: run Insert and Search jobs run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_HOST=localhost \ - E2E_BIND_PORT=8081 \ - E2E_PORTFORWARD_ENABLED=false \ - E2E_DATASET_NAME=${{ env.DATASET }} \ + make hack/benchmark/assets/dataset/${DATASET} + podname=`kubectl get pods --selector=app=vald-lb-gateway | tail -1 | awk '{print $1}'` + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=5000 \ - E2E_SEARCH_COUNT=5000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${podname} \ + E2E_TARGET_NAMESPACE=default \ e2e/insert/search - + env: + DATASET: fashion-mnist-784-euclidean.hdf5 agent-network-partition: name: "E2E chaos test (agent network partition: to test retries)" - needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags with: - ingress_port: ${{ env.INGRESS_PORT }} - - - name: Deploy Vald + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + options: "--image docker.io/rancher/k3s:latest" + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: ${{ env.VALUES }} + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-chaos.yaml wait_for_selector: app=vald-lb-gateway - - - name: Deploy Chaos Mesh - uses: ./.github/actions/deploy-chaos-mesh + - name: deploy Chaos Mesh + run: | + make kubectl/install + curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s + helm install \ + --set networkChaos.partition.enabled=true \ + vald-chaos-test tests/chaos/chart + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 with: - helm_extra_options: "--set networkChaos.partition.enabled=true" - - - name: Run Insert and Search operations + go-version: ${{ steps.golang_version.outputs.version }} + - name: run Insert and Search jobs run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_HOST=localhost \ - E2E_BIND_PORT=8081 \ - E2E_PORTFORWARD_ENABLED=false \ - E2E_DATASET_NAME=${{ env.DATASET }} \ + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ E2E_TIMEOUT=15m \ E2E_INSERT_COUNT=10000 \ E2E_SEARCH_COUNT=10000 \ E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ e2e/insert/search - + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} clusterwide-network-bandwidth: name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)" - needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos' steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags with: - ingress_port: ${{ env.INGRESS_PORT }} - - - name: Deploy Vald + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + options: "--image docker.io/rancher/k3s:latest" + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: ${{ env.VALUES }} + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway - - - name: Deploy Chaos Mesh - uses: ./.github/actions/deploy-chaos-mesh + - name: deploy Chaos Mesh + run: | + make kubectl/install + curl -sSL "https://mirrors.chaos-mesh.org/v$(cat versions/CHAOS_MESH_VERSION)/install.sh" | bash -s -- --k3s + helm install \ + --set networkChaos.bandwidth.enabled=true \ + vald-chaos-test tests/chaos/chart + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 with: - helm_extra_options: "--set networkChaos.bandwidth.enabled=true" - - - name: Run Insert and Search operations + go-version: ${{ steps.golang_version.outputs.version }} + - name: run Insert jobs run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_HOST=localhost \ - E2E_BIND_PORT=8081 \ - E2E_PORTFORWARD_ENABLED=false \ - E2E_DATASET_NAME=${{ env.DATASET }} \ + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ E2E_TIMEOUT=15m \ - E2E_INSERT_COUNT=5000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=5m \ - e2e/insert/search - + E2E_INSERT_COUNT=10000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/insert + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + - name: run Search jobs + run: | + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_TIMEOUT=15m \ + E2E_SEARCH_COUNT=10000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/search + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} slack-notification: name: "Slack notification" - if: startsWith( github.ref, 'refs/tags/') - runs-on: ubuntu-latest needs: - agent-failure - random-pod-failure - agent-network-partition - clusterwide-network-bandwidth + runs-on: ubuntu-latest + if: startsWith( github.ref, 'refs/tags/') steps: - - uses: actions/checkout@v3 - - - uses: ./.github/actions/notify-slack + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 with: - author_name: E2E Chaos test - slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + author_name: "E2E chaos test" + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml index 0a10a19fe0..a996e58921 100644 --- a/.github/workflows/e2e-code-bench-agent.yaml +++ b/.github/workflows/e2e-code-bench-agent.yaml @@ -42,36 +42,55 @@ on: - "versions/GO_VERSION" - "versions/NGT_VERSION" -env: - DATASET: fashion-mnist-784-euclidean.hdf5 - DATASET_ARGS: fashion-mnist - jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} grpc-sequential: name: grpc-sequential runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 - - - name: Set Git config + - name: Check out code. + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - uses: ./.github/actions/setup-go - - name: Run grpc-sequential run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - DATASET_ARGS=${{ env.DATASET_ARGS }} make bench/agent/sequential/grpc - + make hack/benchmark/assets/dataset/${DATASET} + DATASET_ARGS=${DATASET_ARGS} make bench/agent/sequential/grpc + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + DATASET_ARGS: fashion-mnist - name: Upload artifact uses: actions/upload-artifact@v3 with: @@ -90,19 +109,18 @@ jobs: container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 - - - name: Set Git config + - name: Check out code. + uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - uses: ./.github/actions/setup-go - - name: Run grpc-stream run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - DATASET_ARGS=${{ env.DATASET_ARGS }} make bench/agent/stream - + make hack/benchmark/assets/dataset/${DATASET} + DATASET_ARGS=${DATASET_ARGS} make bench/agent/stream + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + DATASET_ARGS: fashion-mnist - name: Upload artifact uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/e2e-deploy.yml b/.github/workflows/e2e-deploy.yml new file mode 100644 index 0000000000..57678708f7 --- /dev/null +++ b/.github/workflows/e2e-deploy.yml @@ -0,0 +1,495 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Run E2E deploy and integration test" +on: + push: + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + pull_request: + types: + - "labeled" +jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} + e2e-stream-crud: + name: "E2E test (Stream CRUD)" + runs-on: ubuntu-latest + timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-stream-crud-for-operator: + name: "E2E test (Stream CRUD) for operator" + runs-on: ubuntu-latest + timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: Merge image tag + run: | + IMAGE_TAGS=(${{ steps.specify_container_versions.outputs.IMAGE_TAGS }}) + + for IMAGE_TAG in "${IMAGE_TAGS[@]}" + do + echo "IMAGE_TAG:${IMAGE_TAG}" + + ARR=(${IMAGE_TAG//=/ }) + FIELD=${ARR[0]} + TAG="\"${ARR[1]}\"" + yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml + done + - name: deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald-helm-operator + with: + require_libhdf5: "true" + valdrelease: ./.github/valdrelease/valdrelease.yaml + wait_for_selector: app=vald-lb-gateway + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + + e2e-stream-crud-skip-exist-check: + name: "E2E test (Stream CRUD: skip strict exist check)" + runs-on: ubuntu-latest + timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=10 \ + E2E_SEARCH_COUNT=10 \ + E2E_SEARCH_BY_ID_COUNT=10 \ + E2E_GET_OBJECT_COUNT=10 \ + E2E_UPDATE_COUNT=10 \ + E2E_UPSERT_COUNT=10 \ + E2E_REMOVE_COUNT=10 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/skip + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-multiapis-crud: + name: "E2E test (Multi-APIs CRUD)" + runs-on: ubuntu-latest + timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-lb.yaml + wait_for_selector: app=vald-lb-gateway + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=100 \ + E2E_SEARCH_COUNT=10 \ + E2E_SEARCH_BY_ID_COUNT=10 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/multi + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-agent-and-sidecar: + name: "E2E Agent & Sidecar test" + runs-on: ubuntu-latest + timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + steps: + - uses: actions/checkout@v3 + - name: set git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + with: + images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libhdf5-dev + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald + with: + require_libhdf5: "true" + require_minio: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-agent-sidecar.yaml + wait_for_selector: app=vald-agent-ngt + wait_for_timeout: 29m + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: run E2E Agent & Sidecar + run: | + make hack/benchmark/assets/dataset/${DATASET} + go version + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${DATASET} \ + E2E_INSERT_COUNT=10000 \ + E2E_SEARCH_COUNT=4000 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e/sidecar + env: + DATASET: fashion-mnist-784-euclidean.hdf5 + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + slack-notification: + name: "Slack notification" + needs: + - e2e-agent-and-sidecar + - e2e-multiapis-crud + - e2e-stream-crud + - e2e-stream-crud-for-operator + - e2e-stream-crud-skip-exist-check + runs-on: ubuntu-latest + if: startsWith( github.ref, 'refs/tags/') + steps: + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 + with: + author_name: "E2E deploy test" + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml index de8f5f8c61..701dd343ed 100644 --- a/.github/workflows/e2e-max-dim.yml +++ b/.github/workflows/e2e-max-dim.yml @@ -26,32 +26,90 @@ on: - "labeled" jobs: - dump-contexts-to-log: - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} e2e-max-dimension-insert: name: "E2E test (Max Dimension Insert: skip strict exist check)" - needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 - + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim' steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - - name: Run E2E MaxDimension + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 1 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: run E2E MaxDimension + run: | + go version go get github.com/vdaas/vald-client-go/v1/payload go get github.com/vdaas/vald-client-go/v1/vald BIT=18 @@ -109,16 +167,21 @@ jobs: WAIT_FOR_TIMEOUT: 29m VALUES: .github/helm/values/values-max-dim.yaml timeout-minutes: 60 - slack-notification: name: "Slack notification" - needs: [e2e-max-dimension-insert] + needs: + - e2e-max-dimension-insert runs-on: ubuntu-latest if: startsWith( github.ref, 'refs/tags/') steps: - - uses: actions/checkout@v3 - - - uses: ./.github/actions/notify-slack + - uses: technote-space/workflow-conclusion-action@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: 8398a7/action-slack@v3 with: author_name: "E2E max dim test" - slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} + status: ${{ env.WORKFLOW_CONCLUSION }} + only_mention_fail: channel + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml index 319f098204..d8faf01554 100644 --- a/.github/workflows/e2e-profiling.yml +++ b/.github/workflows/e2e-profiling.yml @@ -25,53 +25,107 @@ on: types: - "labeled" -env: - DATASET: fashion-mnist-784-euclidean.hdf5 - jobs: - dump-contexts-to-log: - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} e2e-profiling: name: "E2E profiling" - needs: [dump-contexts-to-log] runs-on: ubuntu-latest timeout-minutes: 60 + if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - - name: Deploy Vald + - name: Wait for Docker images + if: startsWith( github.ref, 'refs/tags/') + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Get PR number + id: get_pr_number + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' + run: | + pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` + echo "PR_NUM=${pr_num}" >> $GITHUB_OUTPUT + - name: Specify container versions + if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + tag_name: pr-${{ steps.get_pr_number.outputs.PR_NUM }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-profile.yaml wait_for_selector: app=vald-lb-gateway - - - name: Deploy profefe + - name: deploy profefe run: | make k8s/metrics/profefe/deploy kubectl patch cronjob kprofefe -p '{"spec": {"schedule": "*/1 * * * *"}}' kubectl wait --for=condition=ready pod -l app=profefe --timeout=300s - - - name: Run E2E CRUD + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: run E2E CRUD continue-on-error: true run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} + make hack/benchmark/assets/dataset/${DATASET} make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_DATASET_NAME=${DATASET} \ E2E_INSERT_COUNT=10000 \ E2E_SEARCH_COUNT=100 \ E2E_SEARCH_BY_ID_COUNT=100 \ @@ -84,8 +138,8 @@ jobs: E2E_TARGET_NAMESPACE=default \ e2e env: + DATASET: fashion-mnist-784-euclidean.hdf5 POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - - name: Get profiles run: | mkdir -p profiles @@ -99,24 +153,20 @@ jobs: --output profiles/${svc}-${t}.pb done done - - uses: actions/upload-artifact@v3 with: name: profiles path: profiles/* - - uses: actions/cache@v2 id: cache with: path: ./profiles-main key: ${{ runner.os }}-profiles-main-${{ github.sha }} restore-keys: ${{ runner.os }}-profiles-main- - - name: Install dependencies run: | sudo apt update sudo apt install -y graphviz - - name: Generate graphs run: | tag=$(cat profiles-main/VALD_VERSION || echo "unknown") @@ -139,19 +189,16 @@ jobs: fi done done - - uses: actions/upload-artifact@v3 with: name: graphs path: graphs/* - - name: Update cache if: startsWith( github.ref, 'refs/tags/') run: | mkdir -p profiles-main cp -f profiles/* profiles-main/ cp -f versions/VALD_VERSION profiles-main/ - - name: Upload to vald-ci-images repository if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' run: | @@ -167,7 +214,6 @@ jobs: env: GITHUB_USER: ${{ secrets.DISPATCH_USER }} GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - - name: Comment if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling' run: | diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml deleted file mode 100644 index 180f83cd54..0000000000 --- a/.github/workflows/e2e.yml +++ /dev/null @@ -1,288 +0,0 @@ -# -# Copyright (C) 2019-2023 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Run E2E deploy and integration test" -on: - push: - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - types: - - "labeled" -env: - DATASET: fashion-mnist-784-euclidean.hdf5 - -jobs: - dump-contexts-to-log: - if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-deploy' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - - e2e-stream-crud: - name: "E2E test (Stream CRUD)" - needs: [dump-contexts-to-log] - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml - wait_for_selector: app=vald-lb-gateway - - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10000\ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - - e2e-stream-crud-for-operator: - name: "E2E test (Stream CRUD) for operator" - needs: [dump-contexts-to-log] - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - - name: Create Helm options for vald-helm-operator - uses: ./.github/actions/setup-e2e - id: vald_helm_operator - with: - target_images: "vdaas/vald-helm-operator" - require_libhdf5: false - require_go: false - require_helm: false - require_k3d: false - - - name: Merge Docker image tag - run: | - IMAGE_TAGS=(${{ steps.setup_e2e.outputs.IMAGE_TAGS }}) - - for IMAGE_TAG in "${IMAGE_TAGS[@]}" - do - echo "IMAGE_TAG:${IMAGE_TAG}" - - ARR=(${IMAGE_TAG//=/ }) - FIELD=${ARR[0]} - TAG="\"${ARR[1]}\"" - yq e ".spec.$FIELD=$TAG" -i ./.github/valdrelease/valdrelease.yaml - done - - - name: Deploy Vald using Vald Helm Operator - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald-helm-operator - with: - helm_extra_options: ${{ steps.vald_helm_operator.outputs.HELM_EXTRA_OPTIONS }} - valdrelease: ./.github/valdrelease/valdrelease.yaml - wait_for_selector: app=vald-lb-gateway - - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=10000 \ - E2E_SEARCH_BY_ID_COUNT=10000 \ - E2E_GET_OBJECT_COUNT=100 \ - E2E_UPDATE_COUNT=100 \ - E2E_UPSERT_COUNT=100 \ - E2E_REMOVE_COUNT=100 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - - e2e-stream-crud-skip-exist-check: - name: "E2E test (Stream CRUD: skip strict exist check)" - needs: [dump-contexts-to-log] - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml - wait_for_selector: app=vald-lb-gateway - - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10 \ - E2E_SEARCH_COUNT=10 \ - E2E_SEARCH_BY_ID_COUNT=10 \ - E2E_GET_OBJECT_COUNT=10 \ - E2E_UPDATE_COUNT=10 \ - E2E_UPSERT_COUNT=10 \ - E2E_REMOVE_COUNT=10 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/skip - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - - e2e-multiapis-crud: - name: "E2E test (Multi-APIs CRUD)" - needs: [dump-contexts-to-log] - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-lb.yaml - wait_for_selector: app=vald-lb-gateway - - - name: Run E2E CRUD - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=100 \ - E2E_SEARCH_COUNT=10 \ - E2E_SEARCH_BY_ID_COUNT=10 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=2m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/multi - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - - e2e-agent-and-sidecar: - name: "E2E Agent & Sidecar test" - needs: [dump-contexts-to-log] - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - - name: Set Git config - run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - with: - target_images: vdaas/vald-agent-ngt vdaas/vald-agent-sidecar - - - name: Deploy Vald - id: deploy_vald - uses: ./.github/actions/e2e-deploy-vald - with: - require_minio: "true" - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} - values: .github/helm/values/values-agent-sidecar.yaml - wait_for_selector: app=vald-agent-ngt - wait_for_timeout: 29m - - - name: Run E2E Agent & Sidecar - run: | - make hack/benchmark/assets/dataset/${{ env.DATASET }} - make E2E_BIND_PORT=8081 \ - E2E_DATASET_NAME=${{ env.DATASET }} \ - E2E_INSERT_COUNT=10000 \ - E2E_SEARCH_COUNT=4000 \ - E2E_WAIT_FOR_CREATE_INDEX_DURATION=8m \ - E2E_TARGET_POD_NAME=${POD_NAME} \ - E2E_TARGET_NAMESPACE=default \ - e2e/sidecar - env: - POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} - - slack-notification: - name: "Slack notification" - if: startsWith( github.ref, 'refs/tags/') - needs: - - e2e-agent-and-sidecar - - e2e-multiapis-crud - - e2e-stream-crud - - e2e-stream-crud-for-operator - - e2e-stream-crud-skip-exist-check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: ./.github/actions/notify-slack - with: - author_name: "E2E deploy test" - slack_notify_webhook_url: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }} diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 66a6dda040..083a064214 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -18,36 +18,55 @@ on: push: branches: - main - jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} format: name: Run formatter runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 + - name: Check out code. + uses: actions/checkout@v3 with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - - name: Switch new branch + - name: switch new branch id: switch_to_new_branch run: | TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) @@ -55,16 +74,19 @@ jobs: git checkout main git checkout -b ${BRANCH_NAME} echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_OUTPUT - - - name: Setup Go environment - uses: ./.github/actions/setup-go - + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} - name: Run formatter and license.go run: | make deps/install make format git checkout go.mod go.sum - - name: Check and Push to main branch continue-on-error: true run: | diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index 7f2a7f227a..72658c978c 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -23,30 +23,48 @@ on: - "labeled" jobs: - dump-contexts-to-log: - if: github.ref == 'refs/heads/main' || github.event.action == 'labeled' && github.event.label.name == 'actions/fossa' + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} fossa-scan: name: "trigger FOSSA scan" - needs: [dump-contexts-to-log] runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly + if: github.ref == 'refs/heads/main' || github.event.action == 'labeled' && github.event.label.name == 'actions/fossa' steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: "Install fossa-cli" run: | curl -H 'Cache-Control: no-cache' https://github.com/raw/fossas/fossa-cli/master/install.sh | bash - - name: "Run for main branch" if: github.ref == 'refs/heads/main' run: | @@ -55,7 +73,6 @@ jobs: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_SHA: ${{ github.sha }} - - name: "Run for PRs" if: github.event_name == 'pull_request' run: | diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml index fd4bf9e2bd..7836e5f88d 100644 --- a/.github/workflows/helm-lint.yml +++ b/.github/workflows/helm-lint.yml @@ -20,12 +20,34 @@ on: - "charts/**" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} lint-vald-chart: name: lint for vald chart runs-on: ubuntu-latest @@ -34,23 +56,27 @@ jobs: steps: - name: Check out code. uses: actions/checkout@v3 - - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - uses: ./.github/actions/setup-go - - - uses: ./.github/actions/setup-helm + - name: Helm version + run: | + helm version + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} - name: Generate schema for charts/vald run: | make helm/schema/vald - - name: Run lint for charts/vald run: | helm lint charts/vald - lint-vald-helm-operator-chart: name: lint for vald-helm-operator chart runs-on: ubuntu-latest @@ -59,46 +85,60 @@ jobs: steps: - name: Check out code. uses: actions/checkout@v3 - - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - uses: ./.github/actions/setup-go - - - uses: ./.github/actions/setup-helm + - name: Helm version + run: | + helm version + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} - name: Generate schema for charts/vald-helm-operator run: | make helm/schema/vald-helm-operator - - name: Run lint for charts/vald-helm-operator run: | helm lint charts/vald-helm-operator - lint-values-schema: name: lint for values schema runs-on: ubuntu-latest steps: - name: Check out code. uses: actions/checkout@v3 - - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - uses: ./.github/actions/setup-yq - - - uses: ./.github/actions/setup-go - - - uses: ./.github/actions/setup-helm - - - uses: ./.github/actions/setup-k3d - + - name: Helm version + run: | + helm version + - name: install YQ + run: | + sudo make yq/install + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} - name: Generate ValdRelease schema run: | make helm/schema/crd/vald - + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + - name: check k3d + run: | + kubectl cluster-info - name: Test to apply run: | kubectl create -f charts/vald-helm-operator/crds/valdrelease.yaml diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 0698e52b74..5bad5d0ea5 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -23,36 +23,56 @@ on: - "v*.*.*-*" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} update-k8s-manifest: name: Update k8s manifest runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 + - name: Check out code. + uses: actions/checkout@v3 with: fetch-depth: 0 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - - name: Setup Helm environment - uses: ./.github/actions/setup-helm - - - name: Switch new branch + - name: Helm version + run: | + helm version + - name: switch new branch id: switch_to_new_branch run: | TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) @@ -60,17 +80,14 @@ jobs: git checkout main git checkout -b ${BRANCH_NAME} echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_OUTPUT - - name: Update k8s-dir run: | make k8s/manifest/update make k8s/manifest/helm-operator/update - - name: Run formatter and license.go run: | make format/yaml make license - - name: Push to main continue-on-error: true run: | @@ -92,36 +109,32 @@ jobs: GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} API_URL: https://github.com/gitapi/repos/vdaas/vald/pulls BRANCH_NAME: ${{ steps.switch_to_new_branch.outputs.BRANCH_NAME }} - update-helm-chart: name: Update Helm chart runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 + - name: Check out code. + uses: actions/checkout@v3 with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - - name: Setup Helm environment - uses: ./.github/actions/setup-helm - + - name: Helm version + run: | + helm version - name: Packaging run: | make helm/package/vald make helm/package/vald-helm-operator - - name: Push to gh-pages run: | git checkout . @@ -137,34 +150,63 @@ jobs: env: GITHUB_USER: ${{ secrets.DISPATCH_USER }} GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - crud-on-remote-helm-chart: name: CRUD test on remote Helm chart runs-on: ubuntu-latest timeout-minutes: 60 steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Setup E2E environment - id: setup_e2e - uses: ./.github/actions/setup-e2e - - - name: Deploy Vald + - name: Wait for Docker images + id: wait_for_docker_images + uses: ./.github/actions/wait-for-docker-image + - name: Specify container versions + id: specify_container_versions + uses: ./.github/actions/detect-docker-image-tags + with: + tag_name: ${{ github.ref_name }} + - name: Fetch Helm version + run: | + HELM_VERSION=`make version/helm` + echo "helm=${HELM_VERSION}" >> $GITHUB_OUTPUT + id: version + - uses: rinx/setup-k3d@v0.0.4 + with: + version: latest + name: vald + agents: 3 + - name: check k3d + run: | + kubectl cluster-info + - uses: azure/setup-helm@v3 + with: + version: ${{ steps.version.outputs.helm }} + - name: Helm version + run: | + helm version + - name: deploy Vald id: deploy_vald uses: ./.github/actions/e2e-deploy-vald with: - helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + require_libhdf5: "true" + helm_extra_options: ${{ steps.specify_container_versions.outputs.HELM_EXTRA_OPTIONS }} values: .github/helm/values/values-lb.yaml wait_for_selector: app=vald-lb-gateway use_local_charts: false - - - name: Run E2E CRUD + - name: Fetch golang version + run: | + GO_VERSION=`make version/go` + echo "version=${GO_VERSION}" >> $GITHUB_OUTPUT + id: golang_version + - uses: actions/setup-go@v3 + with: + go-version: ${{ steps.golang_version.outputs.version }} + - name: run E2E CRUD run: | make hack/benchmark/assets/dataset/${DATASET} + go version make E2E_BIND_PORT=8081 \ E2E_DATASET_NAME=${DATASET} \ E2E_INSERT_COUNT=60000 \ diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 616e7284e2..1f22df3926 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -18,12 +18,34 @@ on: - pull_request jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} triage: name: Triage runs-on: ubuntu-latest @@ -32,16 +54,14 @@ jobs: - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" - - - uses: actions/checkout@v3 + - name: Checkout PR branch + uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} fetch-depth: 0 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Add labels run: | pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"` @@ -95,7 +115,6 @@ jobs: env: REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - remove-duplication: name: Remove duplication needs: diff --git a/.github/workflows/reviewdog-hadolint.yml b/.github/workflows/reviewdog-hadolint.yml index 94cb0179d2..6d8403b3f6 100644 --- a/.github/workflows/reviewdog-hadolint.yml +++ b/.github/workflows/reviewdog-hadolint.yml @@ -14,30 +14,50 @@ # limitations under the License. # -name: "Reviewdog - Dockerfiles" +name: "reviewdog - Dockerfiles" on: pull_request: paths: - "dockers/**" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} hadolint: name: runner / hadolint runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Run hadolint + - name: hadolint uses: reviewdog/action-hadolint@v1 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reviewdog-k8s.yml b/.github/workflows/reviewdog-k8s.yml index 12f90a9164..5d233404df 100644 --- a/.github/workflows/reviewdog-k8s.yml +++ b/.github/workflows/reviewdog-k8s.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: "Reviewdog - K8s manifests" +name: "reviewdog - K8s manifests" on: pull_request: paths: @@ -22,12 +22,34 @@ on: - "k8s/**" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} kubeval-conflint: name: runner / kubeval-conflint runs-on: ubuntu-latest @@ -35,12 +57,10 @@ jobs: image: mumoshu/conflint:latest steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Run kubeval-conflint + - name: kubeval-conflint run: | set -vx conflint run -c .github/conflint.yaml -efm "%f:%l:%c: %m" || true @@ -49,7 +69,6 @@ jobs: CONFLINT_LOG: DEBUG REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} REPORTER: github-pr-review - kubelinter-k8s-manifests: name: runner / kubelinter runs-on: ubuntu-latest @@ -57,12 +76,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Run kubelinter for k8s directory + - name: kubelinter for k8s directory run: | kube-linter lint \ --config .github/kubelinter.yaml \ @@ -70,7 +87,6 @@ jobs: k8s/discoverer \ k8s/gateway \ k8s/manager - kubelinter-vald-chart: name: runner / kubelinter for vald chart runs-on: ubuntu-latest @@ -78,12 +94,10 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Run kubelinter for generated manifests + - name: kubelinter for generated manifests run: | kube-linter lint \ --config .github/kubelinter.yaml \ diff --git a/.github/workflows/reviewdog-markdown.yml b/.github/workflows/reviewdog-markdown.yml index c6dda495eb..ad84dc315f 100644 --- a/.github/workflows/reviewdog-markdown.yml +++ b/.github/workflows/reviewdog-markdown.yml @@ -14,7 +14,7 @@ # limitations under the License. # -name: "Reviewdog - Markdown" +name: "reviewdog - Markdown" on: pull_request: paths-ignore: @@ -22,22 +22,42 @@ on: - "CHANGELOG.md" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - - languagetool-apis-charts: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} + languagetool_apis_charts: name: runner / LanguageTool / apis & charts runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: LanguageTool uses: reviewdog/action-languagetool@v1 with: @@ -48,17 +68,14 @@ jobs: language: en-US disabled_rules: "DOUBLE_PUNCTUATION,WORD_CONTAINS_UNDERSCORE,ARROWS,CURRENCY,DASH_RULE,EN_QUOTES" disabled_categories: "TYPOS,TYPOGRAPHY,STYLE,CASING" - - languagetool-docs: + languagetool_docs: name: runner / LanguageTool / Docs runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: LanguageTool uses: reviewdog/action-languagetool@v1 with: @@ -69,26 +86,22 @@ jobs: language: en-US disabled_rules: "DOUBLE_PUNCTUATION,WORD_CONTAINS_UNDERSCORE,ARROWS,CURRENCY,DASH_RULE,EN_QUOTES" disabled_categories: "TYPOS,TYPOGRAPHY,STYLE" - textlint: name: runner / textlint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - - name: Set Git config + - name: Checkout + uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Setup node/npm uses: actions/setup-node@v3 with: node-version: "16" - - name: Install textlint run: | make textlint/ci/install - - name: textlint-github-pr-review uses: tsuyoshicho/action-textlint@v3 with: diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml index ea4c4132b9..c0777b7d4f 100644 --- a/.github/workflows/reviewdog.yml +++ b/.github/workflows/reviewdog.yml @@ -14,19 +14,41 @@ # limitations under the License. # -name: "Reviewdog - Go" +name: reviewdog on: pull_request: paths: - "**.go" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} golangci-lint: name: runner / golangci-lint runs-on: ubuntu-latest @@ -34,15 +56,14 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - uses: actions/checkout@v3 - - - name: Set Git config + with: + persist-credentials: false + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Go build + - name: go build run: | go build -mod=readonly ./... - - name: Run golangci-lint run: | golangci-lint run --config .golangci.yml \ diff --git a/.github/workflows/semver.yml b/.github/workflows/semver.yml index 0ea3150ab8..8a1063b5d3 100644 --- a/.github/workflows/semver.yml +++ b/.github/workflows/semver.yml @@ -20,33 +20,53 @@ on: - main jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} semver-auto: name: Run semver-auto runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 + - name: Check out code. + uses: actions/checkout@v3 with: fetch-depth: 0 token: ${{ secrets.DISPATCH_TOKEN }} - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - name: Get latest gitwerk run: | curl -fsSLO \ @@ -55,7 +75,6 @@ jobs: unzip gitwerk-linux-amd64.zip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Run gitwerk semver-auto id: semver run: | @@ -137,7 +156,6 @@ jobs: env: GITHUB_USER: ${{ secrets.DISPATCH_USER }} GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }} - - name: Create release if: steps.semver.outputs.RELEASE == 'true' id: create_release @@ -151,7 +169,6 @@ jobs: See [CHANGELOG.md](https://github.com/vdaas/vald/blob/${{ steps.semver.outputs.VERSION }}/CHANGELOG.md) for details. draft: false prerelease: false - - name: Add the version to goproxy run: | curl "https://proxy.golang.org/github.com/vdaas/vald/@v/${{ steps.semver.outputs.VERSION }}.info" diff --git a/.github/workflows/test-hack.yml b/.github/workflows/test-hack.yml index 1eff12949e..54c4c929b3 100644 --- a/.github/workflows/test-hack.yml +++ b/.github/workflows/test-hack.yml @@ -30,24 +30,41 @@ on: paths: - "go.mod" - "go.sum" - - ".github/workflows/test-hack.yml" + - ".github/workflows/test.yml" - "hack/gorules/**" - "hack/helm/**" - "hack/license/**" - "hack/tools/**" -env: - # NOTE: This variable is used to make gorules run properly. - GOPATH: ${{ github.workspace }} - PROJECT_ROOT_DIR: src/github.com/vdaas/vald - jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} test-hack: name: Run tests for hack packages runs-on: ubuntu-latest @@ -55,24 +72,25 @@ jobs: image: ghcr.io/vdaas/vald/vald-ci-container:nightly defaults: run: - working-directory: ${{ env.GOPATH }}/${{ env.PROJECT_ROOT_DIR }} + working-directory: src/github.com/vdaas/vald steps: - - uses: actions/checkout@v3 + - name: Check out code. + uses: actions/checkout@v3 with: - path: ${{ env.PROJECT_ROOT_DIR }} - - - name: Set Git config + fetch-depth: 1 + path: src/github.com/vdaas/vald + - name: set git config run: | - git config --global --add safe.directory ${GITHUB_WORKSPACE} - - - name: Install Proto dependencies + git config --global --add safe.directory ${GITHUB_WORKSPACE}/src/github.com/vdaas/vald + working-directory: src/github.com/vdaas/vald + - name: dependencies run: | - make proto/deps - + GOPATH=${GITHUB_WORKSPACE} make proto/deps + working-directory: src/github.com/vdaas/vald - name: Run tests for hack packages / gotestfmt run: | - TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/hack/gotestfmt - - - name: Print tparse result + GOPATH=${GITHUB_WORKSPACE} TEST_RESULT_DIR=${GITHUB_WORKSPACE}/src/github.com/vdaas/vald make test/hack/gotestfmt + working-directory: src/github.com/vdaas/vald + - name: print tparse result run: | - tparse -notests -smallscreen -sort cover -format markdown -file ${GITHUB_WORKSPACE}/test-hack-gotestfmt-result.json + tparse -notests -smallscreen -sort cover -format markdown -file ${GITHUB_WORKSPACE}/src/github.com/vdaas/vald/test-hack-gotestfmt-result.json diff --git a/.github/workflows/unit-test.yaml b/.github/workflows/test.yml similarity index 70% rename from .github/workflows/unit-test.yaml rename to .github/workflows/test.yml index 3f96d047ea..9f17b061a9 100644 --- a/.github/workflows/unit-test.yaml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Run unit tests" +name: "Run tests" on: push: branches: @@ -35,68 +35,82 @@ on: - "cmd/**" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} test-cmd: name: Run tests for cmd packages runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 - - - name: Set Git config + - name: Check out code. + uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Run tests for cmd packages / gotestfmt run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/cmd/gotestfmt - - - name: Print tparse result + - name: print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-cmd-gotestfmt-result.json" - test-internal: name: Run tests for internal packages runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 - - - name: Set Git config + - name: Check out code. + uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Run tests for internal packages / gotestfmt run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/internal/gotestfmt - - - name: Print tparse result + - name: print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-internal-gotestfmt-result.json" - test-pkg: name: Run tests for pkg packages runs-on: ubuntu-latest container: image: ghcr.io/vdaas/vald/vald-ci-container:nightly steps: - - uses: actions/checkout@v3 - - - name: Set Git config + - name: Check out code. + uses: actions/checkout@v3 + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - name: Run tests for pkg packages / gotestfmt run: | TEST_RESULT_DIR=${GITHUB_WORKSPACE} make test/pkg/gotestfmt - - - name: Print tparse result + - name: print tparse result run: | tparse -notests -smallscreen -sort cover -format markdown -file "${GITHUB_WORKSPACE}/test-pkg-gotestfmt-result.json" diff --git a/.github/workflows/update-protobuf.yaml b/.github/workflows/update-protobuf.yaml deleted file mode 100644 index 41aa2fda1d..0000000000 --- a/.github/workflows/update-protobuf.yaml +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright (C) 2019-2023 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Update protobuf" -on: - workflow_dispatch: - push: - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - -jobs: - dump-contexts-to-log: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - - client-go: - uses: ./.github/workflows/_update-protobuf.yaml - with: - repo: vald-client-go - secrets: inherit - - client-java: - uses: ./.github/workflows/_update-protobuf.yaml - with: - repo: vald-client-java - secrets: inherit - - client-node: - uses: ./.github/workflows/_update-protobuf.yaml - with: - repo: vald-client-node - secrets: inherit - - client-python: - uses: ./.github/workflows/_update-protobuf.yaml - with: - repo: vald-client-python - secrets: inherit diff --git a/.github/workflows/update-protobuf.yml b/.github/workflows/update-protobuf.yml new file mode 100644 index 0000000000..67455803f3 --- /dev/null +++ b/.github/workflows/update-protobuf.yml @@ -0,0 +1,85 @@ +# +# Copyright (C) 2019-2023 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: update protobuf +on: + push: + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + +jobs: + dump_contexts_to_log: + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} + dispatches: + name: Dispatches + runs-on: ubuntu-latest + steps: + - name: client-go + run: | + curl --fail -u "${USER}:${TOKEN}" -X POST https://github.com/gitapi/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' + env: + REPO: vdaas/vald-client-go + USER: ${{ secrets.DISPATCH_USER }} + TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: client-java + run: | + curl --fail -u "${USER}:${TOKEN}" -X POST https://github.com/gitapi/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' + env: + REPO: vdaas/vald-client-java + USER: ${{ secrets.DISPATCH_USER }} + TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: client-python + run: | + curl --fail -u "${USER}:${TOKEN}" -X POST https://github.com/gitapi/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' + env: + REPO: vdaas/vald-client-python + USER: ${{ secrets.DISPATCH_USER }} + TOKEN: ${{ secrets.DISPATCH_TOKEN }} + - name: client-node + run: | + curl --fail -u "${USER}:${TOKEN}" -X POST https://github.com/gitapi/repos/${REPO}/dispatches -H 'Accept: application/vnd.github.everest-preview+json' --data '{"event_type": "update-protobuf"}' + env: + REPO: vdaas/vald-client-node + USER: ${{ secrets.DISPATCH_USER }} + TOKEN: ${{ secrets.DISPATCH_TOKEN }} diff --git a/.github/workflows/update-pull-request-and-issue-template.yml b/.github/workflows/update_pull_request_and_issue_template.yml similarity index 82% rename from .github/workflows/update-pull-request-and-issue-template.yml rename to .github/workflows/update_pull_request_and_issue_template.yml index 4273b17b63..ac7ba7fa86 100644 --- a/.github/workflows/update-pull-request-and-issue-template.yml +++ b/.github/workflows/update_pull_request_and_issue_template.yml @@ -24,30 +24,50 @@ on: - "versions/KUBECTL_VERSION" jobs: - dump-contexts-to-log: + dump_contexts_to_log: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/dump-context - + - name: Dump GitHub context + id: github_context_step + run: echo $JSON + env: + JSON: ${{ toJSON(github) }} + - name: Dump job context + run: echo $JSON + env: + JSON: ${{ toJSON(job) }} + - name: Dump steps context + run: echo $JSON + env: + JSON: ${{ toJSON(steps) }} + - name: Dump runner context + run: echo $JSON + env: + JSON: ${{ toJSON(runner) }} + - name: Dump strategy context + run: echo $JSON + env: + JSON: ${{ toJSON(strategy) }} + - name: Dump matrix context + run: echo $JSON + env: + JSON: ${{ toJSON(matrix) }} update-pr-issue-template: name: Update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Check out code. + uses: actions/checkout@v3 with: fetch-depth: 0 - - - name: Set Git config + - name: set git config run: | git config --global --add safe.directory ${GITHUB_WORKSPACE} - - uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} git_user_signingkey: true git_commit_gpgsign: true - - name: Push to main run: | TIMESTAMP=$(date +%Y%m%d_%H%M%S_%3N) diff --git a/Makefile b/Makefile index 50ca6a7874..ec122ea034 100644 --- a/Makefile +++ b/Makefile @@ -504,10 +504,6 @@ version/kind: version/helm: @echo $(HELM_VERSION) -.PHONY: version/yq -version/yq: - @echo $(YQ_VERSION) - .PHONY: version/valdcli version/valdcli: @echo $(VALDCLI_VERSION) diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index 515137f54a..18b29ed43a 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -109,7 +109,6 @@ k8s/vald-helm-operator/deploy: helm template \ --output-dir $(TEMP_DIR) \ --set image.tag=$(VERSION) \ - $(HELM_EXTRA_OPTIONS) \ --include-crds \ charts/vald-helm-operator kubectl create -f $(TEMP_DIR)/vald-helm-operator/crds/valdrelease.yaml diff --git a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go index eda3bf6570..e4c5e7d832 100644 --- a/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go +++ b/tests/e2e/pkg/agent/core/ngt/service/ngt_e2s_test.go @@ -66,9 +66,9 @@ func registerVector(ctx context.Context, n service.NGT) error { for i := int64(0); i < maxIDNum; i++ { uuid := strconv.FormatInt(i, 10) - vec, _, err := n.GetObject(uuid) - if err != nil || len(vec) == 0 { - return errors.ErrObjectNotFound(err, uuid) + _, err := n.GetObject(uuid) + if err != nil { + return err } } return nil @@ -150,9 +150,9 @@ func Test_ngt_parallel_delete_and_insert(t *testing.T) { for i := int64(0); i < maxIDNum; i++ { uuid := strconv.FormatInt(i, 10) - vec, _, err := n.GetObject(uuid) - if err != nil || len(vec) == 0 { - t.Error(errors.ErrObjectNotFound(err, uuid)) + _, err := n.GetObject(uuid) + if err != nil { + t.Error(err) } err = n.Insert(uuid, []float32{1, 2}) if err == nil {