Add shell tests to CI (#3039)

.github/workflows/ci-workflow.yml

@@ -417,7 +417,8 @@ jobs:
 
   macos-build-test:
     name: apple clang build & test
-    needs: [clang-formatting-check, sanity-checks, rustfmt-check, python-lint-check]
+    needs:
+      [clang-formatting-check, sanity-checks, rustfmt-check, python-lint-check]
     runs-on: self-hosted-mac-x64
     env:
       NUM_THREADS: 32
@@ -487,3 +488,21 @@ jobs:
           ulimit -n 10240
           source /Users/runner/.cargo/env
           cargo build --locked --features arrow
+
+  shell-test:
+    name: shell test
+    runs-on: ubuntu-latest
+    needs: [clang-formatting-check, sanity-checks]
+    env:
+      WERROR: 0
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build
+        run: make release NUM_THREADS=$(nproc)
+
+      - name: Test
+        working-directory: tools/shell/test
+        run: |
+          pip3 install pytest pexpect
+          python3 -m pytest -v .