.gitlab-ci.yml

image: aistorage/ci:latest

stages:
  - build
  - test-short
  - test-long
  - security

include:
  - project: pstooling/gitlab-templates
    ref: main
    file: /templates/static-analysis/Checkmarx-main-csv.gitlab-ci.yml

variables:
  MODE: debug # run aistore and tests with debug asserts enabled
  SCRIPTS_DIR: "./scripts"
  NUM_TARGET:
    value: "5"
    description: "Number of targets deployed."
  NUM_PROXY:
    value: "5"
    description: "Number of proxies deployed."
  FS_CNT:
    value: "6"
    description: "Number of disks defined for each target."
  CLOUD_BCKS:
    value: "aws://ais-blossom gs://ais-ci-kube" # NOTE: additionally used: aws://ais-cicd and aws://ais-ci-python
    description: "Set of buckets used when running cloud tests."
  GOOGLE_APPLICATION_CREDENTIALS:
    value: "$GCP_JSON_FILE"
    description: "Gitlab file variable containing credentials for GCP"


# Templates

.gather_logs_template: &gather_logs_def
  after_script:
    - make kill # To make sure that nodes flushed the logs.
    - mkdir $CI_PROJECT_DIR/logs
    - find /tmp/ais -type f -name "*\.INFO\.*" -exec cp {} $CI_PROJECT_DIR/logs/ \;
  artifacts:
    when: on_failure
    paths: [ logs/ ]
    expire_in: 1 days

.default_only_template: &default_only_def
  only:
    - main
    - merge_requests
    - schedules
    - webs

.test_short_template: &test_short_def
  stage: test-short
  tags:
    - ais
  timeout: 30m
  <<: *default_only_def
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
      - $CI_MERGE_REQUEST_LABELS =~ /.*python-tests-only.*/
  <<: *gather_logs_def

.test_short_skip_scheduled_template: &test_short_skip_scheduled_def
  stage: test-short
  tags:
    - ais
  timeout: 30m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def


.test_short_optional_template: &test_short_optional_def
  stage: test-short
  tags:
    - ais
  timeout: 30m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

.test_long_template: &test_long_def
  stage: test-long
  tags:
    - ais
  # NOTE: when changing, make sure to update $SCRIPTS_DIR/bootstrap.sh and GitLab /settings/ci_cd
  timeout: 4h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

.test_long_skip_scheduled_template: &test_long_skip_scheduled_def
  stage: test-long
  tags:
    - ais
  # NOTE: when changing, make sure to update $SCRIPTS_DIR/bootstrap.sh and GitLab /settings/ci_cd
  timeout: 4h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  <<: *gather_logs_def

# Stages

build:linux:
  stage: build
  tags:
    - ais
  timeout: 10m
  <<: *default_only_def
  script:
    - MODE="" make node # Build node without backends in production mode.
    - MODE="debug" make node # Build node without backends in debug mode.
    - AIS_BACKEND_PROVIDERS="aws azure gcp" MODE="" make node # Build with all Cloud backends (production mode).
    - AIS_BACKEND_PROVIDERS="aws azure gcp ht" MODE="debug" make node # Build with all backends (debug mode).
    - MEM_PROFILE="/tmp/mem" CPU_PROFILE="/tmp/cpu" make node # Build with profile.
    - TAGS="nethttp" make node # Build with net/http transport support (fasthttp is used by default).
    - make authn
    - make cli
    - make aisloader

build:k8s:
  stage: build
  image: quay.io/buildah/stable
  variables:
    # Use vfs with buildah. Docker offers overlayfs as a default, but Buildah
    # cannot stack overlayfs on top of another overlayfs filesystem.
    STORAGE_DRIVER: vfs
    # Write all image metadata in the docker format, not the standard OCI format.
    BUILDAH_FORMAT: docker
    FQ_IMAGE_NAME: "$CI_REGISTRY_IMAGE/aisnode:$CI_COMMIT_SHORT_SHA"

  before_script:
    - buildah login -u $CI_REGISTRY_USER -p $CI_REGISTRY_TOKEN $CI_REGISTRY
  script:
    - buildah images
    - buildah build -t $FQ_IMAGE_NAME -f deploy/dev/k8s/Dockerfile --build-arg MODE="debug" --build-arg providers="gcp"
    - buildah images
    - buildah push $FQ_IMAGE_NAME
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      allow_failure: false
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*etl.*/'
      allow_failure: false
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
      allow_failure: false
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/'
      when: never

lint:linux:
  stage: build
  tags:
    - ais
  timeout: 10m
  <<: *default_only_def
  script:
    - make lint
    - TAGS="statsd ht" make lint
    - make fmt-check
    - make spell-check

# Runs cluster with 5 proxies and 5 targets (each with 6 mountpaths).
test:short:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - make test-short
    - FLAGS="--duration=10s" make test-aisloader

# Runs cluster with 5 proxies and 5 targets (each with 6 mountpaths).
test:short:python:
  <<: *test_short_def
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
    BUCKET: "aws://ais-ci-python"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --aws
    - cd python
    - make python_sdk_tests
    - make python_botocore_tests

  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/

test:short:pytorch:
  <<: *test_short_def
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
    BUCKET: "aws://ais-ci-python"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --aws
    - cd python
    - make python_pytorch_tests
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
      - $CI_MERGE_REQUEST_LABELS !~ /.*pytorch.*/

# Runs cluster with 1 proxy and 1 target (with 6 mountpaths).
test:short:minimal:
  <<: *test_short_def
  variables:
    BUCKET: "ais://test"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt 1 --proxy-cnt 1 --mountpath-cnt $FS_CNT --deployment all
    - NUM_PROXY=1 NUM_TARGET=1 make test-short

test:short:s3-compat:
  <<: *test_short_def
  tags:
    - ais
  variables:
    # Run only s3 compat tests we know should pass
    S3_COMPAT_RUN_ALL:
      value: "False"
    # Fail on first failure
    S3_COMPAT_STRICT:
      value: "True"

  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all
    - ais config cluster features S3-API-via-Root
    - ais config cluster checksum.type=md5
    - cd python
    - make python_s3_compat_test
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS !~ /.*s3-compat.*/
      - $CI_PIPELINE_SOURCE == "schedule"


test:short:authn:
  <<: *test_short_optional_def
  variables:
    AIS_AUTHN_ENABLED: "true"
    AIS_AUTHN_SU_NAME: "admin"
    AIS_AUTHN_SU_PASS: "admin"
    AIS_AUTHN_URL: "http://localhost:52001"
    BUCKET: "ais://test"
    RE: "TestAuth"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - ais auth login $AIS_AUTHN_SU_NAME -p $AIS_AUTHN_SU_PASS
    - make test-run
    - ais auth logout

test:short:https:
  <<: *test_short_optional_def
  variables:
    AIS_USE_HTTPS: "true"
    AIS_SKIP_VERIFY_CRT: "true"
    AIS_SERVER_CRT: "$CI_PROJECT_DIR/localhost.crt"
    AIS_SERVER_KEY: "$CI_PROJECT_DIR/localhost.key"
    AIS_ENDPOINT: "https://localhost:8080"
    BUCKET: "ais://ais-ci"
  script:
    - openssl req -x509 -out $AIS_SERVER_CRT -keyout $AIS_SERVER_KEY -newkey rsa:2048 -nodes -sha256 -subj '/CN=localhost' -extensions EXT -config <( printf "[dn]\nCN=localhost\n[req]\ndistinguished_name = dn\n[EXT]\nsubjectAltName=DNS:localhost\nkeyUsage=digitalSignature\nextendedKeyUsage=serverAuth")
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --deployment all --https
    - make test-short

test:short:s3rproxy:
  <<: *test_short_skip_scheduled_def
  variables:
    BUCKET: "ais://ais-ci"
    RE: "S3"
  script:
    - deploy/scripts/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - ais config cluster features S3-Reverse-Proxy
    - make test-short

test:short:aws:
  <<: *test_short_skip_scheduled_def
  variables:
    BUCKET: "aws://ais-cicd"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws
    - make test-short

test:short:gcp:
  <<: *test_short_skip_scheduled_def
  variables:
    BUCKET: "gs://ais-ci"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp
    - make test-short

test:short:azure:
  <<: *test_short_skip_scheduled_def
  variables:
    BUCKET: "az://ais-ci"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure
    - make test-short

test:long:
  <<: *test_long_skip_scheduled_def
  variables:
    NUM_PROXY: 6
    BUCKET: "ais://ais-ci"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --ht
    - make test-long

test:long:aws:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "aws://ais-cicd"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --aws --ht
    - make test-long

test:long:gcp:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "gs://ais-ci"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --gcp --ht
    - make test-long

test:long:azure:
  <<: *test_long_def
  variables:
    NUM_PROXY: 6
    BUCKET: "az://ais-ci"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT --azure --ht
    - make test-long

test:long:aisloader:
  stage: test-long
  tags:
    - ais
  timeout: 15m
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # make sure that cluster properly starts
    - FLAGS="--duration=5m" make test-aisloader
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true

test:long:pyaisloader:
  stage: test-long
  tags:
    - ais
  timeout: 15m
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # make sure that cluster properly starts
    - cd ./python; make PYAISLOADER_TEST_TYPE=long test-pyaisloader
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true

test:long:ishard:
  stage: test-long
  tags:
    - ais
  timeout: 1h
  variables:
    AIS_ENDPOINT: "http://localhost:8080"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - sleep 10 # make sure that cluster properly starts
    - cd ./tools/ishard; go test -v -timeout=1h -run=^TestIshard
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true

#
# Kubernetes stages
#

.test_k8s:
  tags:
    - ais-k8s
  variables:
    BUCKET: "gs://ais-ci-kube"
    TESTS_DIR: "ais/test"
    AISNODE_IMAGE: "$CI_REGISTRY_IMAGE/aisnode:$CI_COMMIT_SHORT_SHA"
    AIS_ENDPOINT: "http://ais-proxy-0.default.svc.cluster.local:8080"
    FS_CNT: 6
    RE: "TestETL|TestConfig|TestMountpath"
    PROVIDERS: gcp
  before_script:
    - kubectl delete pods,services -l nvidia.com/ais-etl-name # TODO: this can be removed once the lifecycle of transformers is implemented.
    - cd deploy/dev/k8s
    - make deploy-ci
    - cd ../../..
    - make cli
  after_script:
    - mkdir $CI_PROJECT_DIR/logs
    - find /tmp/ais -type f -name "*\.INFO\.*" -exec cp {} $CI_PROJECT_DIR/logs/ \;
    - cd deploy/dev/k8s
    - make cleanup-ci
  artifacts:
    when: on_failure
    paths: [ logs/ ]
    expire_in: 1 days

.test_k8s_short_template:
  stage: test-short
  extends: .test_k8s
  variables:
    NUM_PROXY: 1
    NUM_TARGET: 1

.test_k8s_long_template:
  stage: test-long
  extends: .test_k8s
  variables:
    NUM_PROXY: 1
    NUM_TARGET: 5

test:short:k8s:
  extends: .test_k8s_short_template
  timeout: 1h
  only:
    - merge_requests
    - schedules
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
      - $CI_MERGE_REQUEST_LABELS =~ /.*python-tests-only.*/
      - $CI_MERGE_REQUEST_LABELS !~ /.*etl.*/
  variables:
    RE: "TestETL|TestConfig|TestMountpath"
  script:
    - make test-short

test:short:python-authn:
  <<: *test_short_optional_def
  variables:
    AIS_AUTHN_ENABLED: "true"
    AIS_AUTHN_SU_NAME: "admin"
    AIS_AUTHN_SU_PASS: "admin"
    AIS_AUTHN_URL: "http://localhost:52001"
  script:
    - ${SCRIPTS_DIR}/clean_deploy.sh --target-cnt $NUM_TARGET --proxy-cnt $NUM_PROXY --mountpath-cnt $FS_CNT
    - cd python/
    - make python_authn_tests

test:short:python-etl:
  extends: .test_k8s_short_template
  <<: *default_only_def
  except:
    variables:
      - $CI_MERGE_REQUEST_LABELS =~ /.*skip-ci.*/
      - $CI_MERGE_REQUEST_LABELS !~ /.*etl.*/
  retry:
    max: 2
    when:
      - unknown_failure
      - api_failure
      - stuck_or_timeout_failure
      - runner_system_failure
      - job_execution_timeout
  script:
    - cd python
    - make python_etl_tests
    

#  e.g. RE: "ETLBucket|ETLConnectionError|ETLInitCode" (or any other regex to select tests)
test:short:assorted:k8s:
  extends: .test_k8s_short_template
  timeout: 30m
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  variables:
    RE: "ETLAnyToAny|ETLMultiObj"
  script:
    - make test-run

test:long:k8s:
  extends: .test_k8s_long_template
  ## NOTE: when changing, make sure to update $SCRIPTS_DIR/bootstrap.sh and GitLab /settings/ci_cd
  timeout: 4h
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "web"'
      when: manual
      allow_failure: true
  script:
    - make test-run

test:long:k8s:single-target:
  extends: .test_k8s_long_template
  timeout: 4h
  rules:
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  variables:
    NUM_TARGET: 1
  script:
    - make test-run

test:long:k8s:aisloader:
  extends: .test_k8s_long_template
  timeout: 15m
  rules:
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  script:
    - sleep 10 # Give some time for the cluster to stabilize.
    - make aisloader
    - BUCKET="ais://test" FLAGS="--duration=2m --etl" make test-aisloader

test:long:k8s:pyaisloader:
  extends: .test_k8s_long_template
  timeout: 15m
  rules:
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  script:
    - sleep 10 # Give some time for the cluster to stabilize.
    - cd ./python; make PYAISLOADER_TEST_TYPE=etl test-pyaisloader

test:long:k8s:all:
  extends: .test_k8s_long_template
  timeout: 5h
  rules:
    - if: '$CI_MERGE_REQUEST_LABELS =~ /.*k8s-ci.*/'
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || $CI_COMMIT_BRANCH == "main"'
      when: manual
      allow_failure: true
  variables: 
    PROVIDERS: "gcp,aws"
  before_script:
    - kubectl delete pods,services -l nvidia.com/ais-etl-name # TODO: this can be removed once the lifecycle of transformers is implemented.
    - cd deploy/dev/k8s
    - make deploy-ci
    - cd ../../..
    # Make sure that metrics collection is enabled.
    - git clone https://github.com/prometheus-operator/kube-prometheus.git
    - kubectl apply -f kube-prometheus/manifests/setup && kubectl apply -f kube-prometheus/manifests && rm -rf kube-prometheus
  script:
    - |
      echo "----- RUNNING K8S TESTS -----"
      BUCKET="aws://ais-blossom" RE="TestETL|TestConfig|TestMountpath" make test-run
      exit_code=$?
      result=$((result + exit_code))
      echo "----- K8S TESTS FINISHED WITH: ${exit_code} -----"
    - |
      for bucket in ${CLOUD_BCKS}; do
        echo "----- RUNNING LONG TESTS WITH: ${bucket} -----"
        BUCKET=${bucket} make test-long && make test-aisloader
        exit_code=$?
        result=$((result + exit_code))
        echo "----- LONG TESTS FINISHED WITH: ${exit_code} -----"
      done

checkmarx-scan-csv:
  stage: security
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
      allow_failure: true