Merge branch 'main' into zongyu/sqlsmith_subquery

risingwavelabs · Aug 1, 2022 · 04f4b12 · 04f4b12
2 parents 1244669 + 161cb22
commit 04f4b12
Show file tree

Hide file tree

Showing 460 changed files with 15,485 additions and 9,092 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -28,6 +28,7 @@ members = [
   "src/stream",
   "src/test_runner",
   "src/tests/regress",
+  "src/tests/simulation",
   "src/tests/sqlsmith",
   "src/utils/memcomparable",
   "src/utils/pgwire",

diff --git a/Makefile.toml b/Makefile.toml
@@ -5,6 +5,7 @@ extend = [
   { path = "src/risedevtool/etcd.toml" },
   { path = "src/risedevtool/jaeger.toml" },
   { path = "src/risedevtool/kafka.toml" },
+  { path = "src/risedevtool/redis.toml" },
   { path = "src/risedevtool/risedev-components.toml" },
   { path = "src/frontend/test_runner/planner_test.toml" },
 ]
@@ -271,6 +272,7 @@ dependencies = [
   "download-prometheus",
   "download-jaeger",
   "download-kafka",
+  "download-redis",
   "build-risingwave",
   "post-build-risingwave",
   "extract-dashboard-artifact",
@@ -289,6 +291,7 @@ dependencies = [
   "download-prometheus",
   "download-jaeger",
   "download-kafka",
+  "download-redis",
 ]
 
 [tasks.p]
@@ -366,9 +369,6 @@ category = "RiseDev - Stop"
 description = "Kill RisingWave dev cluster and clean the data"
 dependencies = ["k", "clean-data"]
 
-[tasks.fill-kafka]
-command = "scripts/source/prepare_source.sh"
-
 [tasks.install-tools]
 category = "RiseDev - Check"
 script = """
@@ -391,10 +391,6 @@ echo "check: $(tput setaf 4)tmux >= v3.2a$(tput sgr0)"
 tmux -V || echo "$(tput setaf 3)tmux$(tput sgr0) not found."
 echo
 
-echo "check: $(tput setaf 4)yq >= 4$(tput sgr0)"
-yq -V || echo "$(tput setaf 3)yq$(tput sgr0) not found."
-echo
-
 echo "check: $(tput setaf 4)psql >= 14$(tput sgr0)"
 psql -V || echo "$(tput setaf 3)psql$(tput sgr0) not found."
 echo
@@ -443,7 +439,7 @@ script = """
 #!/bin/bash
 set -e
 
-cargo build --workspace --exclude risingwave_cmd --exclude risingwave_cmd_all --exclude risingwave_rt --exclude risingwave_frontend_test_runner "$@"
+cargo build "$@"
 """
 
 [tasks.stest]
@@ -455,7 +451,7 @@ script = """
 #!/bin/bash
 set -e
 
-cargo nextest run --workspace --exclude risingwave_cmd --exclude risingwave_cmd_all --exclude risingwave_rt --exclude risingwave_frontend_test_runner "$@"
+cargo test "$@"
 """
 
 [tasks.check-hakari]

diff --git a/README.md b/README.md
@@ -20,20 +20,18 @@ There are two ways to install RisingWave: use a pre-built package or compile fro
 
 ```shell
 # Download the pre-built binary
-wget https://github.com/singularity-data/risingwave/releases/download/v0.1.10/risingwave-v0.1.10-x86_64-unknown-linux.tar.gz
+wget https://github.com/singularity-data/risingwave/releases/download/v0.1.11/risingwave-v0.1.11-x86_64-unknown-linux.tar.gz
 # Unzip the binary
-tar xvf risingwave-v0.1.10-x86_64-unknown-linux.tar.gz
+tar xvf risingwave-v0.1.11-x86_64-unknown-linux.tar.gz
 # Start RisingWave in single-binary playground mode
 ./risingwave playground
 ```
 
 **Use Docker (Linux, macOS)**
 
-Currently, only x86_64 is supported. We will provide arm64 builds in the future.
-
 ```shell
 # Start RisingWave in single-binary playground mode
-docker run -it --pull=always -p 4566:4566 -p 5691:5691 ghcr.io/singularity-data/risingwave:v0.1.10 playground
+docker run -it --pull=always -p 4566:4566 -p 5691:5691 ghcr.io/singularity-data/risingwave:v0.1.11 playground
 ```
 
 **Compile from Source with [RiseDev](docs/developer-guide.md#set-up-the-development-environment) (Linux and macOS)**

diff --git a/ci/README.md b/ci/README.md
@@ -0,0 +1,12 @@
+# CI Workflow
+
+## Sqlsmith
+
+Sqlsmith has two `cron` workflows.
+1. Frontend tests
+2. E2e tests (TODO)
+
+It is separate from PR workflow because:
+1. Fuzzing tests might fail as new features and generators are added.
+2. Avoid slowing down PR workflow, fuzzing tests take a while to run.
+   We can include failing tests in e2e / unit tests when encountered.
diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh
@@ -14,7 +14,7 @@ export RUST_TOOLCHAIN=$(cat ../rust-toolchain)
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 #          AND ALSO docker-compose.yml               #
 ######################################################
-export BUILD_ENV_VERSION=v20220722
+export BUILD_ENV_VERSION=v20220729
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
@@ -15,14 +15,14 @@ services:
       retries: 5
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20220722
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20220729
     volumes:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20220722
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20220729
     depends_on:
       db:
         condition: service_healthy
     volumes:
-      - ..:/risingwave
+      - ..:/risingwave
diff --git a/ci/scripts/deterministic-simulation-test.sh b/ci/scripts/deterministic-simulation-test.sh
@@ -12,4 +12,4 @@ echo "--- Generate RiseDev CI config"
 cp ci/risedev-components.ci.env risedev-components.user.env
 
 echo "--- Run unit tests in deterministic simulation mode"
-cargo make stest --no-fail-fast
+timeout 10m cargo make stest --no-fail-fast
diff --git a/ci/scripts/docker.sh b/ci/scripts/docker.sh
@@ -5,6 +5,7 @@ set -euo pipefail
 
 date="$(date +%Y%m%d)"
 ghcraddr="ghcr.io/singularity-data"
+arch="$(uname -m)"
 
 components=(
   "risingwave"
@@ -17,23 +18,23 @@ components=(
 for component in "${components[@]}"
 do
   echo "--- docker build and tag : ${component}"
-  docker build -f docker/Dockerfile -t "${ghcraddr}/${component}:latest" --target "${component}" .
+  docker build -f docker/Dockerfile -t "${ghcraddr}/${component}:latest-${arch}" --target "${component}" .
   if [ "${BUILDKITE_SOURCE}" == "schedule" ] || [ "${BUILDKITE_SOURCE}" == "ui" ]; then
     # If this is a schedule/ui build, tag the image with the date.
-    TAG="${ghcraddr}/${component}:nightly-${date}"
-    docker tag "${ghcraddr}/${component}:latest" "$TAG"
+    TAG="${ghcraddr}/${component}:nightly-${date}-${arch}"
+    docker tag "${ghcraddr}/${component}:latest-${arch}" "$TAG"
     echo "$TAG"
   fi
   if [[ -n "${BUILDKITE_TAG}" ]]; then
     # If there's a tag, we tag the image.
-    TAG="${ghcraddr}/${component}:${BUILDKITE_TAG}"
-    docker tag "${ghcraddr}/${component}:latest" "$TAG"
+    TAG="${ghcraddr}/${component}:${BUILDKITE_TAG}-${arch}"
+    docker tag "${ghcraddr}/${component}:latest-${arch}" "$TAG"
     echo "$TAG"
   fi
   if [[ "${#BUILDKITE_COMMIT}" = 40 ]]; then
     # If the commit is 40 characters long, it's probably a SHA.
-    TAG="${ghcraddr}/${component}:git-${BUILDKITE_COMMIT}"
-    docker tag "${ghcraddr}/${component}:latest" "$TAG"
+    TAG="${ghcraddr}/${component}:git-${BUILDKITE_COMMIT}-${arch}"
+    docker tag "${ghcraddr}/${component}:latest-${arch}" "$TAG"
     echo "$TAG"
   fi
 done
@@ -48,21 +49,21 @@ if [ "$PUSH_GHCR" = true ]; then
   echo "--- docker push to ghcr"
   for component in "${components[@]}"
   do
-    docker push "${ghcraddr}/${component}:latest"
+    if [[ "${#BUILDKITE_COMMIT}" = 40 ]]; then
+      # If the commit is 40 characters long, it's probably a SHA.
+      TAG="${ghcraddr}/${component}:git-${BUILDKITE_COMMIT}-${arch}"
+      docker push "$TAG"
+    fi
     if [ "${BUILDKITE_SOURCE}" == "schedule" ] || [ "${BUILDKITE_SOURCE}" == "ui" ]; then
       # If this is a schedule/ui build, tag the image with the date.
-      TAG="${ghcraddr}/${component}:nightly-${date}"
+      TAG="${ghcraddr}/${component}:nightly-${date}-${arch}"
       docker push "$TAG"
     fi
     if [[ -n "${BUILDKITE_TAG}" ]]; then
       # If there's a tag, we tag the image.
-      TAG="${ghcraddr}/${component}:${BUILDKITE_TAG}"
-      docker push "$TAG"
-    fi
-    if [[ "${#BUILDKITE_COMMIT}" = 40 ]]; then
-      # If the commit is 40 characters long, it's probably a SHA.
-      TAG="${ghcraddr}/${component}:git-${BUILDKITE_COMMIT}"
+      TAG="${ghcraddr}/${component}:${BUILDKITE_TAG}-${arch}"
       docker push "$TAG"
     fi
+    docker push "${ghcraddr}/${component}:latest-${arch}"
   done
-fi
+fi
diff --git a/ci/scripts/multi-arch-docker.sh b/ci/scripts/multi-arch-docker.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+date="$(date +%Y%m%d)"
+ghcraddr="ghcr.io/singularity-data"
+
+components=(
+  "risingwave"
+  "compute-node"
+  "meta-node"
+  "frontend-node"
+  "compactor-node"
+)
+
+echo "--- ghcr login"
+echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+
+for component in "${components[@]}"
+do
+  echo "--- multi arch image create : ${component}"
+  if [[ "${#BUILDKITE_COMMIT}" = 40 ]]; then
+    # If the commit is 40 characters long, it's probably a SHA.
+    TAG="${ghcraddr}/${component}:git-${BUILDKITE_COMMIT}"
+    docker manifest create --insecure "$TAG" \
+      --amend "${ghcraddr}/${component}:latest-x86_64" \
+      --amend "${ghcraddr}/${component}:latest-aarch64"
+    docker manifest push --insecure "$TAG"
+  fi
+
+  if [ "${BUILDKITE_SOURCE}" == "schedule" ] || [ "${BUILDKITE_SOURCE}" == "ui" ]; then
+    # If this is a schedule/ui build, tag the image with the date.
+    TAG="${ghcraddr}/${component}:nightly-${date}"
+    docker manifest create --insecure "$TAG" \
+      --amend "${ghcraddr}/${component}:latest-x86_64" \
+      --amend "${ghcraddr}/${component}:latest-aarch64"
+    docker manifest push --insecure "$TAG"
+  fi
+
+  if [[ -n "${BUILDKITE_TAG}" ]]; then
+    # If there's a tag, we tag the image.
+    TAG="${ghcraddr}/${component}:${BUILDKITE_TAG}"
+    docker manifest create --insecure "$TAG" \
+      --amend "${ghcraddr}/${component}:latest-x86_64" \
+      --amend "${ghcraddr}/${component}:latest-aarch64"
+    docker manifest push --insecure "$TAG"
+  fi
+
+  TAG="${ghcraddr}/${component}:latest"
+  docker manifest create --insecure "$TAG" \
+    --amend "${ghcraddr}/${component}:latest-x86_64" \
+    --amend "${ghcraddr}/${component}:latest-aarch64"
+  docker manifest push --insecure "$TAG"
+done
diff --git a/ci/scripts/pr-unit-test.sh b/ci/scripts/pr-unit-test.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+source ci/scripts/common.env.sh
+source ci/scripts/pr.env.sh
+./ci/scripts/run-unit-test.sh
diff --git a/ci/scripts/pr.env.sh b/ci/scripts/pr.env.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -euo pipefail
+
+set +e
+# Set features, depending on our workflow
+# If sqlsmith files are modified, we run unit tests with sqlsmith enabled
+# by overriding RUSTFLAGS to enable sqlsmith feature.
+CHANGED=$(git diff --name-only origin/main | grep "ci/scripts/pr.env.sh\|src/tests/sqlsmith")
+set -e
+
+if [[ -n "$CHANGED" ]]; then
+    echo "Changes to Sqlsmith source files detected.";
+    export RUN_SQLSMITH=1;
+    echo "Enabled Sqlsmith tests.";
+else
+# Otherwise we use default.
+    export RUN_SQLSMITH=0;
+fi
diff --git a/ci/scripts/unit-test.sh → ci/scripts/run-unit-test.sh b/ci/scripts/unit-test.sh → ci/scripts/run-unit-test.sh
@@ -1,13 +1,12 @@
 #!/bin/bash
-
-# Exits as soon as any line fails.
 set -euo pipefail
 
-source ci/scripts/common.env.sh
-
 echo "+++ Run unit tests with coverage"
 # use tee to disable progress bar
-NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints 2> >(tee)
+NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints 2> >(tee);
+if [[ "$RUN_SQLSMITH" -eq "1" ]]; then
+    NEXTEST_PROFILE=ci cargo nextest run run_sqlsmith_on_frontend --features "failpoints enable_sqlsmith_unit_test" 2> >(tee);
+fi
 
 echo "--- Codecov upload coverage reports"
 curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov

diff --git a/ci/scripts/unit-fuzz-test.sh b/ci/scripts/unit-fuzz-test.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+source ci/scripts/common.env.sh
+export RUN_SQLSMITH=1; # fuzz tests
+source ci/scripts/run-unit-test.sh
diff --git a/ci/workflows/docker.yml b/ci/workflows/docker.yml
@@ -2,18 +2,12 @@ projects:
   - label: docker-pr
     path: docker/
     skip:
-      - docker-build-push-main
-      - docker-build-push-schedule-ui
-  - label: docker-main
-    path: docker/
-    skip:
-      - docker-build-pr
-      - docker-build-push-schedule-ui
+      - docker-build-push*
+      - multi-arch-image-create-push
   - label: docker-schedule-ui
     path: .
     skip:
       - docker-build-pr
-      - docker-build-push-main
 
 auto-retry: &auto-retry
   automatic:
@@ -31,9 +25,10 @@ steps:
       PUSH_GHCR: false
     retry: *auto-retry
 
-  - label: "docker-build-push-main"
+  - label: "docker-build-push: amd64"
     command: "ci/scripts/docker.sh"
-    if: build.branch == "main" && build.source != "schedule"
+    key: "build-amd64"
+    if: build.source == "schedule" || build.source == "ui"
     env:
       BUILDPIPE_SCOPE: project
       PUSH_GHCR: true
@@ -44,8 +39,9 @@ steps:
             GHCR_TOKEN: ghcr-token
     retry: *auto-retry
 
-  - label: "docker-build-push-schedule-ui"
+  - label: "docker-build-push: aarch64"
     command: "ci/scripts/docker.sh"
+    key: "build-aarch64"
     if: build.source == "schedule" || build.source == "ui"
     env:
       BUILDPIPE_SCOPE: project
@@ -55,4 +51,21 @@ steps:
           env:
             GHCR_USERNAME: ghcr-username
             GHCR_TOKEN: ghcr-token
+    retry: *auto-retry
+    agents:
+      queue: "linux-arm64"
+
+  - label: "multi-arch-image-create-push"
+    command: "ci/scripts/multi-arch-docker.sh"
+    if: build.source == "schedule" || build.source == "ui"
+    depends_on:
+      - "build-amd64"
+      - "build-aarch64"
+    env:
+      BUILDPIPE_SCOPE: project
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GHCR_USERNAME: ghcr-username
+            GHCR_TOKEN: ghcr-token
     retry: *auto-retry