add test

ci/scripts/s3-source-test.sh

@@ -20,6 +20,23 @@ while getopts 'p:' opt; do
 done
 shift $((OPTIND -1))
 
+
+while getopts 's:' opt; do
+    case ${opt} in
+        p )
+            script=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
 echo "--- Download artifacts"
 mkdir -p target/debug
 buildkite-agent artifact download risingwave-"$profile" target/debug/
@@ -44,7 +61,7 @@ cargo make ci-start ci-1cn-1fe
 
 echo "--- Run test"
 python3 -m pip install minio psycopg2-binary
-python3 e2e_test/s3/run.py
+python3 e2e_test/s3/$script.py
 
 echo "--- Kill cluster"
 cargo make ci-kill

ci/workflows/main-cron.yml

@@ -184,8 +184,8 @@ steps:
     timeout_in_minutes: 5
     retry: *auto-retry
 
-  - label: "S3 source check on AWS"
-    command: "ci/scripts/s3-source-test.sh -p ci-release"
+  - label: "S3 source check on AWS (json parser)"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s run"
     depends_on: build
     plugins:
       - seek-oss/aws-sm#v2.3.1:
@@ -200,8 +200,8 @@ steps:
     timeout_in_minutes: 20
     retry: *auto-retry  
 
-  - label: "S3 source check on lyvecloud.seagate.com"
-    command: "ci/scripts/s3-source-test.sh -p ci-release"
+  - label: "S3 source check on lyvecloud.seagate.com (json parser)"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s run"
     depends_on: build
     plugins:
       - seek-oss/aws-sm#v2.3.1:

ci/workflows/pull-request.yml

@@ -43,270 +43,18 @@ steps:
     timeout_in_minutes: 15
     retry: *auto-retry
 
-  - label: "build other components"
-    command: "ci/scripts/build-other.sh"
-    key: "build-other"
+  - label: "S3 source check on AWS (csv parser)"
+    command: "ci/scripts/s3-source-test.sh -p ci-dev -s run_csv"
+    depends_on: build
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
-            GITHUB_TOKEN: github-token
+            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
       - docker-compose#v4.9.0:
           run: rw-build-env
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
           environment:
-            - GITHUB_TOKEN
-    timeout_in_minutes: 10
-    retry: *auto-retry
-
-  - label: "build (deterministic simulation)"
-    command: "ci/scripts/build-simulation.sh"
-    key: "build-simulation"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-    timeout_in_minutes: 15
-    retry: *auto-retry
-
-  - label: "docslt"
-    command: "ci/scripts/docslt.sh"
-    key: "docslt"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-    timeout_in_minutes: 10
-    retry: *auto-retry
-
-  - label: "end-to-end test"
-    command: "ci/scripts/e2e-test.sh -p ci-dev"
-    depends_on:
-      - "build"
-      - "docslt"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 12
-    retry: *auto-retry
-
-  - label: "end-to-end test (parallel)"
-    command: "ci/scripts/e2e-test-parallel.sh -p ci-dev"
-    depends_on:
-      - "build"
-      - "docslt"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 12
-    retry: *auto-retry
-
-  - label: "end-to-end test (parallel, in-memory)"
-    command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-dev"
-    depends_on: "build"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 12
-    retry: *auto-retry
-
-  - label: "end-to-end source test"
-    command: "ci/scripts/e2e-source-test.sh -p ci-dev"
-    depends_on:
-      - "build"
-      - "build-other"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: source-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 18
-    retry: *auto-retry
-
-  - label: "end-to-end sink test"
-    command: "ci/scripts/e2e-sink-test.sh -p ci-dev"
-    depends_on:
-      - "build"
-      - "build-other"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: sink-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 5
-    retry: *auto-retry
-
-  - label: "end-to-end iceberg sink test"
-    command: "ci/scripts/e2e-iceberg-sink-test.sh -p ci-dev"
-    depends_on:
-      - "build"
-      - "build-other"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 5
-    retry: *auto-retry
-
-  - label: "e2e java-binding test"
-    command: "ci/scripts/java-binding-test.sh -p ci-dev"
-    depends_on: "build"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 5
-    retry: *auto-retry
-
-  - label: "regress test"
-    command: "ci/scripts/regress-test.sh -p ci-dev"
-    depends_on: "build"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: regress-test-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 5
-    retry: *auto-retry
-
-  - label: "unit test"
-    command: "ci/scripts/pr-unit-test.sh"
-    plugins:
-      - ./ci/plugins/swapfile
-      - gencer/cache#v2.4.10: *cargo-cache
-      - seek-oss/aws-sm#v2.3.1:
-          env:
-            CODECOV_TOKEN: my-codecov-token
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          environment:
-            - CODECOV_TOKEN
-    timeout_in_minutes: 12
-    retry: *auto-retry
-
-  - label: "fuzz test"
-    command: "ci/scripts/pr-fuzz-test.sh -p ci-dev"
-    depends_on:
-      - "build"
-      - "build-simulation"
-    plugins:
-      - ./ci/plugins/swapfile
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 15
-    retry: *auto-retry
-
-  - label: "check"
-    command: "ci/scripts/check.sh"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
+            - S3_SOURCE_TEST_CONF
     timeout_in_minutes: 20
-    retry: *auto-retry
-
-  - label: "unit test (deterministic simulation)"
-    command: "ci/scripts/deterministic-unit-test.sh"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-    timeout_in_minutes: 10
-    retry: *auto-retry
-
-  - label: "scaling test (deterministic simulation)"
-    command: "TEST_NUM=5 ci/scripts/deterministic-scale-test.sh"
-    depends_on: "build-simulation"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-    timeout_in_minutes: 10
-    retry: *auto-retry
-
-  - label: "end-to-end test (deterministic simulation)"
-    command: "TEST_NUM=16 timeout 14m ci/scripts/deterministic-e2e-test.sh"
-    depends_on: "build-simulation"
-    plugins:
-      - seek-oss/aws-sm#v2.3.1:
-          env:
-            GITHUB_TOKEN: github-token
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-          environment:
-            - GITHUB_TOKEN
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 15
-    retry: *auto-retry
-
-  - label: "recovery test (deterministic simulation)"
-    command: "TEST_NUM=8 KILL_RATE=0.5 timeout 14m ci/scripts/deterministic-recovery-test.sh"
-    depends_on: "build-simulation"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      # - seek-oss/aws-sm#v2.3.1:
-      #     env:
-      #       BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-deterministic-token
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      # - test-collector#v1.0.0:
-      #     files: "*-junit.xml"
-      #     format: "junit"
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 15
-    retry: *auto-retry
-
-  - label: "misc check"
-    command: "ci/scripts/misc-check.sh"
-    plugins:
-      - gencer/cache#v2.4.10: *cargo-cache
-      - docker-compose#v4.9.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-      - shellcheck#v1.2.0:
-          files: ./**/*.sh
-    timeout_in_minutes: 5
-    retry: *auto-retry
+    retry: *auto-retry