Merge branches 'w/2.8/improvement/ZENKO-4773-end2end-pra-archive-test…

…s' and 'q/2114/2.7/improvement/ZENKO-4773-end2end-pra-archive-tests' into tmp/octopus/q/2.8

.github/actions/deploy/action.yaml

@@ -7,6 +7,10 @@ inputs:
     description: "The tag of the Zenko Operator image to use"
     required: false
     default: ""
+  extra_components:
+    description: "Extra components to add to zenkoversion"
+    required: false
+    default: ""
 runs:
   using: composite
   steps:
@@ -74,6 +78,8 @@ runs:
       shell: bash
       run: bash deploy-zenko.sh end2end default
       working-directory: ./.github/scripts/end2end
+      env:
+        EXTRA_COMPONENTS: ${{ inputs.extra_components }}
     - name: Add Keycloak user and assign StorageManager role
       shell: bash
       run: bash keycloak-helper.sh add-user default

.github/scripts/end2end/configs/keycloak_user.json

@@ -5,7 +5,7 @@
   "emailVerified": true,
   "firstName": "${OIDC_FIRST_NAME}",
   "lastName": "${OIDC_LAST_NAME}",
-  "email": "e2e@zenko.local",
+  "email": "${OIDC_EMAIL}",
   "attributes": {
     "instanceIds": [
         "${INSTANCE_ID}"

.github/scripts/end2end/configs/zenkoversion.yaml

@@ -127,8 +127,10 @@ spec:
         image: "${REDIS_IMAGE}"
         tag: "${REDIS_TAG}"
       exporter:
-        image: "${REDIS_EXPORTER_IMAGE}"
-        tag: "${REDIS_EXPORTER_TAG}"
+        image: '${REDIS_EXPORTER_IMAGE}'
+        tag: '${REDIS_EXPORTER_TAG}'
+      kubedb: '${REDIS_KUBEDB_TAG}'
+    ${EXTRA_COMPONENTS}
   defaults:
     backbeatConcurrency:
       lifecycleBucketProcessor: 30

.github/scripts/end2end/configure-e2e-ctst.sh

@@ -46,9 +46,9 @@ kubectl run kafka-topics \
     --restart=Never \
     --attach=True \
     --command -- bash -c \
-    "kafka-topics.sh --create --topic $NOTIF_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT ; \
-        kafka-topics.sh --create --topic $NOTIF_ALT_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT ; \
-        kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC --partitions 10 --bootstrap-server $KAFKA_HOST_PORT ; \
-        kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_NV --partitions 10 --bootstrap-server $KAFKA_HOST_PORT ; \
-        kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_V --partitions 10 --bootstrap-server $KAFKA_HOST_PORT ; \
-        kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_S --partitions 10 --bootstrap-server $KAFKA_HOST_PORT"
+    "kafka-topics.sh --create --topic $NOTIF_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
+        kafka-topics.sh --create --topic $NOTIF_ALT_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
+        kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
+        kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_NV --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
+        kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_V --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
+        kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_S --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists"

.github/scripts/end2end/configure-e2e.sh

@@ -47,6 +47,27 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
 EOF
 
+KAFKA_REGISTRY_NAME=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yaml)
+KAFKA_IMAGE_NAME=$(yq eval ".kafka.image" ../../../solution/deps.yaml)
+KAFKA_IMAGE_TAG=$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
+KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG
+KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
+    -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
+KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
+
+# Creating replication/transition and notification topics in kafka
+kubectl run kafka-topics \
+    --image=$KAFKA_IMAGE \
+    --pod-running-timeout=5m \
+    --rm \
+    --restart=Never \
+    --attach=True \
+    --command -- bash -c \
+    "kafka-topics.sh --create --topic $UUID.backbeat-replication-replay-0 --partitions 5 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
+    kafka-topics.sh --create --topic $UUID.backbeat-data-mover --partitions 5 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
+    kafka-topics.sh --create --topic $NOTIF_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \
+    kafka-topics.sh --create --topic $NOTIF_ALT_DEST_TOPIC --bootstrap-server $KAFKA_HOST_PORT --if-not-exists "
+
 kubectl run ${POD_NAME} \
   --image ${E2E_IMAGE} \
   --rm \

.github/scripts/end2end/keycloak-helper.sh

@@ -6,6 +6,7 @@ DIR=$(dirname "$0")
 
 COMMAND=${1:-''}
 NAMESPACE=${2:-default}
+ZENKO_NAME=${3:-end2end}
 
 KEYCLOAK_EXEC="kubectl -n ${NAMESPACE} exec -i keycloak-0 --"
 
@@ -28,7 +29,9 @@ case $COMMAND in
     "add-user")
         refresh_creds
 
-        export INSTANCE_ID=`kubectl -n ${NAMESPACE} get zenko -o jsonpath='{.items[0].status.instanceID}'`
+        export INSTANCE_ID=`kubectl -n ${NAMESPACE} get zenko ${ZENKO_NAME} -o jsonpath='{.status.instanceID}'`
+
+        export OIDC_EMAIL=${OIDC_EMAIL:-"e2e@zenko.local"}
 
         envsubst < $DIR/configs/keycloak_user.json | \
             ${KEYCLOAK_EXEC} /opt/jboss/keycloak/bin/kcadm.sh create users -r ${OIDC_REALM} -f -

.github/scripts/end2end/patch-coredns.sh

@@ -27,6 +27,13 @@ corefile="
     rewrite name exact sts.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact iam.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact shell-ui.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact keycloak.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact ui.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact management.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact s3.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact sts.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact iam.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact shell-ui.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     kubernetes cluster.local in-addr.arpa ip6.arpa {
         pods insecure
         fallthrough in-addr.arpa ip6.arpa

.github/scripts/end2end/prepare-pra.sh

@@ -6,11 +6,14 @@ export MONGODB_PRA_DATABASE="${MONGODB_PRA_DATABASE:-'pradb'}"
 export ZENKO_MONGODB_DATABASE="${MONGODB_PRA_DATABASE}"
 export ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"
 
-export ZENKO_IAM_INGRESS="iam.zenko-pra.local"
-export ZENKO_STS_INGRESS="sts.zenko-pra.local"
-export ZENKO_MANAGEMENT_INGRESS="management.zenko-pra.local"
-export ZENKO_S3_INGRESS="s3.zenko-pra.local"
-export ZENKO_UI_INGRESS="ui.zenko-pra.local"
+echo 'ZENKO_MONGODB_DATABASE="pradb"' >> $GITHUB_ENV
+echo 'ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"' >> $GITHUB_ENV
+
+echo 'ZENKO_IAM_INGRESS="iam.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_STS_INGRESS="sts.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_MANAGEMENT_INGRESS="management.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_S3_INGRESS="s3.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_UI_INGRESS="ui.dr.zenko.local"' >> $GITHUB_ENV
 
 MONGODB_ROOT_USERNAME="${MONGODB_ROOT_USERNAME:-'root'}"
 MONGODB_ROOT_PASSWORD="${MONGODB_ROOT_PASSWORD:-'rootpass'}"

.github/scripts/end2end/run-e2e-ctst.sh

@@ -5,6 +5,9 @@ ZENKO_NAME=${1:-end2end}
 COMMAND=${2:-"premerge"}
 PARALLEL_RUNS=${PARALLEL_RUNS:-$(( ( $(nproc) + 1 ) / 2 ))}
 RETRIES=${4:-3}
+
+shift 4
+
 JUNIT_REPORT_PATH=${JUNIT_REPORT_PATH:-"ctst-junit.xml"}
 
 # Zenko Version
@@ -14,6 +17,8 @@ VERSION=$(cat ../../../VERSION | grep -Po 'VERSION="\K[^"]*')
 ZENKO_ACCOUNT_NAME="zenko-ctst"
 ADMIN_ACCESS_KEY_ID=$(kubectl get secret end2end-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
 ADMIN_SECRET_ACCESS_KEY=$(kubectl get secret end2end-management-vault-admin-creds.v1  -o jsonpath='{.data.secretKey}' | base64 -d)
+ADMIN_PRA_ACCESS_KEY_ID=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
+ADMIN_PRA_SECRET_ACCESS_KEY=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1  -o jsonpath='{.data.secretKey}' | base64 -d)
 STORAGE_MANAGER_USER_NAME="ctst_storage_manager"
 STORAGE_ACCOUNT_OWNER_USER_NAME="ctst_storage_account_owner"
 DATA_CONSUMER_USER_NAME="ctst_data_consumer"
@@ -43,6 +48,9 @@ KAFKA_DEAD_LETTER_TOPIC=$(kubectl get secret -l app.kubernetes.io/name=cold-sorb
 KAFKA_OBJECT_TASK_TOPIC=$(kubectl get secret -l app.kubernetes.io/name=cold-sorbet-config-e2e-azure-archive,app.kubernetes.io/instance=end2end \
     -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq '."kafka-object-task-topic"' | cut -d "\"" -f 2)
 
+DR_ADMIN_ACCESS_KEY_ID=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
+DR_ADMIN_SECRET_ACCESS_KEY=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1  -o jsonpath='{.data.secretKey}' | base64 -d)
+
 # Extracting kafka host from bacbeat's config
 KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
     -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
@@ -66,6 +74,7 @@ SORBETD_RESTORE_TIMEOUT=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.so
 WORLD_PARAMETERS="$(jq -c <<EOF
 {
   "subdomain":"${SUBDOMAIN}",
+  "DRSubdomain":"${DR_SUBDOMAIN}",
   "ssl":false,
   "port":"${ZENKO_PORT}",
   "AccountName":"${ZENKO_ACCOUNT_NAME}",
@@ -102,7 +111,10 @@ WORLD_PARAMETERS="$(jq -c <<EOF
   "BackbeatApiHost":"${BACKBEAT_API_HOST}",
   "BackbeatApiPort":"${BACKBEAT_API_PORT}",
   "KafkaCleanerInterval":"${KAFKA_CLEANER_INTERVAL}",
-  "SorbetdRestoreTimeout":"${SORBETD_RESTORE_TIMEOUT}"
+  "SorbetdRestoreTimeout":"${SORBETD_RESTORE_TIMEOUT}",
+  "TimeProgressionFactor":"${TIME_PROGRESSION_FACTOR}",
+  "DRAdminAccessKey":"${DR_ADMIN_ACCESS_KEY_ID}",
+  "DRAdminSecretKey":"${DR_ADMIN_SECRET_ACCESS_KEY}"
 }
 EOF
 )"
@@ -117,8 +129,8 @@ kubectl set env cronjob end2end-ops-count-items PROMETHEUS_POLLING_PERIOD=1
 
 E2E_IMAGE=$E2E_CTST_IMAGE_NAME:$E2E_IMAGE_TAG
 POD_NAME="${ZENKO_NAME}-ctst-tests"
-
 CTST_VERSION=$(sed 's/.*"cli-testing": ".*#\(.*\)".*/\1/;t;d' ../../../tests/ctst/package.json)
+
 # Configure keycloak
 docker run \
     --rm \
@@ -181,4 +193,4 @@ kubectl run $POD_NAME \
       }
     ]
   }
-}' -- ./run "$COMMAND" $WORLD_PARAMETERS --parallel $PARALLEL_RUNS --retry $RETRIES --retry-tag-filter @Flaky --format junit:$JUNIT_REPORT_PATH
+}' -- ./run "$COMMAND" $WORLD_PARAMETERS --parallel $PARALLEL_RUNS --retry $RETRIES --retry-tag-filter @Flaky --format junit:$JUNIT_REPORT_PATH "$@"

.github/workflows/end2end.yaml

@@ -86,6 +86,7 @@ env:
   NOTIF_ALT_DEST_NAME: "destination2"
   NOTIF_ALT_DEST_TOPIC: "destination-topic-2"
   SUBDOMAIN: "zenko.local"
+  DR_SUBDOMAIN: "dr.zenko.local"
   SKOPEO_PATH: "/tmp"
   HELM_VERSION: "v3.5.3"
   YQ_VERSION: "v4.27.5"
@@ -96,6 +97,7 @@ env:
   KUBECTL_VERSION: "1.30.0"
   TILT_VERSION: "0.23.4"
   KIND_VERSION: "v0.20.0"
+  DRCTL_TAG: "v1.0.2"
   ZENKO_ENABLE_SOSAPI: false
   EXPIRE_ONE_DAY_EARLIER: true
   TRANSITION_ONE_DAY_EARLIER: true
@@ -400,6 +402,7 @@ jobs:
           build-args: |
             CTST_TAG=${{ env.CTST_TAG }}
             SORBET_TAG=${{ env.SORBET_TAG }}
+            DRCTL_TAG=${{ env.DRCTL_TAG}}
           tags: "${{ env.E2E_CTST_IMAGE_NAME }}:${{ env.E2E_IMAGE_TAG }}"
           cache-from: type=gha,scope=end2end-ctst
           cache-to: type=gha,mode=max,scope=end2end-ctst
@@ -470,11 +473,8 @@ jobs:
         if: always()
 
   end2end-pra:
-    needs: [build-kafka, build-test-image, check-dashboard-versions]
-    runs-on:
-      - ubuntu
-      - focal
-      - xlarge
+    needs: [build-kafka, check-dashboard-versions, lint-and-build-ctst]
+    runs-on: ubuntu-22.04-16core
     env:
       GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
     steps:
@@ -495,46 +495,41 @@ jobs:
         env:
           ZENKO_MONGODB_SHARDED: "true"
         with:
-          zkop_tag: 1.6.0-preview.1
+          zkop_tag: 1.6.0-preview.2
+          extra_components: |
+            drctl:
+                  image: ghcr.io/scality/zenko-drctl
+                  tag: ${{ env.DRCTL_TAG }}
       - name: Prepare PRA environment
         run: bash prepare-pra.sh
         working-directory: ./.github/scripts/end2end
       - name: Deploy second Zenko for PRA
-        run: bash deploy-zenko.sh end2end-pra default
+        run: bash deploy-zenko.sh end2end-pra default './configs/zenko.yaml'
+        env:
+          ZENKO_MONGODB_SHARDED: "true"
+          ZENKO_MONGODB_DATABASE: "pradb"
+          EXTRA_COMPONENTS: |
+            drctl:
+                  image: 'ghcr.io/scality/zenko-drctl'
+                  tag: ${{ env.DRCTL_TAG }}
         working-directory: ./.github/scripts/end2end
-      - name: Deploy second Sorbet for PRA
-        run: bash deploy-sorbet-resources.sh end2end-pra
-        working-directory: ./.github/scripts/end2end/operator/tests/smoke
+      - name: Add Keycloak pra user and assign StorageManager role
+        shell: bash
+        run: bash keycloak-helper.sh add-user default end2end-pra
         env:
-          SORBETD_NAME: mock-sorbet-pra
-      - name: Extract environment
-        run: |-
-          sudo curl --fail -L https://github.com/mikefarah/yq/releases/download/${{ env.YQ_VERSION}}/${{ env.YQ_BINARY }} -o /usr/bin/yq
-          sudo chmod +x /usr/bin/yq
-
-          get_image_from_deps() {
-              local dep_name=$1
-              yq eval ".$dep_name | (.sourceRegistry // \"docker.io\") + \"/\" + .image" deps.yaml
-          }
-
-          cd solution/
-          cat <<EOF >> $GITHUB_ENV
-          KAFKA_IMAGE=$(get_image_from_deps kafka)
-          KAFKA_TAG=$(yq eval '.kafka.tag' deps.yaml)
-          KAFKA_CONNECT_IMAGE=$(get_image_from_deps kafka-connect)
-          KAFKA_CONNECT_TAG=$(yq eval '.kafka-connect.tag' deps.yaml)
-          JMX_JAVAAGENT_IMAGE=$(get_image_from_deps jmx-javaagent)
-          JMX_JAVAAGENT_TAG=$(yq eval '.jmx-javaagent.tag' deps.yaml)
-          MONGODB_CONNECTOR_TAG=$(yq eval '.mongodb-connector.tag' deps.yaml)
-          KAFKA_CLEANER_IMAGE=$(get_image_from_deps kafka-cleaner)
-          KAFKA_CLEANER_TAG=$(yq eval '.kafka-cleaner.tag' deps.yaml)
-          KAFKA_CRUISECONTROL_IMAGE=$(get_image_from_deps kafka-cruise-control)
-          KAFKA_CRUISECONTROL_TAG=$(yq eval '.kafka-cruise-control.tag' deps.yaml)
-          KAFKA_LAGEXPORTER_IMAGE=$(get_image_from_deps kafka-lag-exporter)
-          KAFKA_LAGEXPORTER_TAG=$(yq eval '.kafka-lag-exporter.tag' deps.yaml)
-          EOF
-      - name: Deploy zenko dr components
-        run: bash deploy-dr.sh
+          OIDC_USERNAME: 'zenko-end2end-pra'
+          OIDC_EMAIL: 'e2e-pra@zenko.local'
+        working-directory: ./.github/scripts/end2end
+      - name: Configure E2E PRA test environment
+        run: bash configure-e2e.sh end2end-pra ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} default
+        working-directory: ./.github/scripts/end2end
+        env:
+          OIDC_USERNAME: 'zenko-end2end-pra'
+      - name: Configure E2E CTST test environment
+        run: bash configure-e2e-ctst.sh
+        working-directory: ./.github/scripts/end2end
+      - name: Run CTST end to end tests
+        run: bash run-e2e-ctst.sh "" "" "" "" --tags @PRA
         working-directory: ./.github/scripts/end2end
       - name: Archive artifact logs and data
         uses: ./.github/actions/archive-artifacts
@@ -710,7 +705,7 @@ jobs:
         run: bash configure-e2e-ctst.sh
         working-directory: ./.github/scripts/end2end
       - name: Run CTST end to end tests
-        run: bash run-e2e-ctst.sh
+        run: bash run-e2e-ctst.sh "" "" "" "" --tags 'not @PRA'
         working-directory: ./.github/scripts/end2end
       - name: Upload results
         if: "!cancelled() && env.TRUNK_TOKEN"

solution/deps.yaml

@@ -90,6 +90,11 @@ sorbet:
   image: sorbet
   tag: v1.1.10
   envsubst: SORBET_TAG
+drctl:
+  sourceRegistry: ghcr.io/scality
+  image: zenko-drctl
+  tag: v1.0.2
+  envsubst: DRCTL_TAG
 # To be enabled back when utapi is used in Zenko 2.x
 # utapi:
 #   sourceRegistry: ghcr.io/scality

tests/ctst/Dockerfile

@@ -1,6 +1,9 @@
 ARG SORBET_TAG
 ARG CTST_TAG
+ARG DRCTL_TAG
+
 FROM ghcr.io/scality/sorbet:$SORBET_TAG as sorbet
+FROM ghcr.io/scality/zenko-drctl:$DRCTL_TAG as drctl
 FROM ghcr.io/scality/cli-testing:$CTST_TAG
 
 COPY package.json /tmp/package.json
@@ -19,5 +22,6 @@ ENV SDK=true
 WORKDIR /ctst
 
 COPY --from=sorbet /sorbetctl .
+COPY --from=drctl /zenko-drctl .
 
 CMD ["./run"]