diff --git a/Dockerfile.cypress b/.ci/Dockerfile.cypress similarity index 100% rename from Dockerfile.cypress rename to .ci/Dockerfile.cypress diff --git a/.ci/compose.ci.yaml b/.ci/compose.ci.yaml new file mode 100644 index 0000000000..7c056d0f26 --- /dev/null +++ b/.ci/compose.ci.yaml @@ -0,0 +1,25 @@ +services: + redash: + build: ../ + command: manage version + depends_on: + - postgres + - redis + ports: + - "5000:5000" + environment: + PYTHONUNBUFFERED: 0 + REDASH_LOG_LEVEL: "INFO" + REDASH_REDIS_URL: "redis://redis:6379/0" + POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb" + REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres" + REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF" + redis: + image: redis:7-alpine + restart: unless-stopped + postgres: + image: pgautoupgrade/pgautoupgrade:latest + command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF" + restart: unless-stopped + environment: + POSTGRES_HOST_AUTH_METHOD: "trust" diff --git a/.ci/compose.cypress.yaml b/.ci/compose.cypress.yaml new file mode 100644 index 0000000000..7f769ab3ef --- /dev/null +++ b/.ci/compose.cypress.yaml @@ -0,0 +1,73 @@ +x-redash-service: &redash-service + build: + context: ../ + args: + install_groups: "main" + code_coverage: ${CODE_COVERAGE} +x-redash-environment: &redash-environment + REDASH_LOG_LEVEL: "INFO" + REDASH_REDIS_URL: "redis://redis:6379/0" + POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb" + REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres" + REDASH_RATELIMIT_ENABLED: "false" + REDASH_ENFORCE_CSRF: "true" + REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF" +services: + server: + <<: *redash-service + command: server + depends_on: + - postgres + - redis + ports: + - "5000:5000" + environment: + <<: *redash-environment + PYTHONUNBUFFERED: 0 + scheduler: + <<: *redash-service + command: scheduler + depends_on: + - server + environment: + <<: *redash-environment + worker: + <<: *redash-service + command: worker + depends_on: + - server + environment: + <<: *redash-environment + PYTHONUNBUFFERED: 0 + cypress: + ipc: host + build: + context: ../ + dockerfile: .ci/Dockerfile.cypress + depends_on: + - server + - worker + - scheduler + environment: + CYPRESS_baseUrl: "http://server:5000" + CYPRESS_coverage: ${CODE_COVERAGE} + PERCY_TOKEN: ${PERCY_TOKEN} + PERCY_BRANCH: ${CIRCLE_BRANCH} + PERCY_COMMIT: ${CIRCLE_SHA1} + PERCY_PULL_REQUEST: ${CIRCLE_PR_NUMBER} + COMMIT_INFO_BRANCH: ${CIRCLE_BRANCH} + COMMIT_INFO_MESSAGE: ${COMMIT_INFO_MESSAGE} + COMMIT_INFO_AUTHOR: ${CIRCLE_USERNAME} + COMMIT_INFO_SHA: ${CIRCLE_SHA1} + COMMIT_INFO_REMOTE: ${CIRCLE_REPOSITORY_URL} + CYPRESS_PROJECT_ID: ${CYPRESS_PROJECT_ID} + CYPRESS_RECORD_KEY: ${CYPRESS_RECORD_KEY} + redis: + image: redis:7-alpine + restart: unless-stopped + postgres: + image: pgautoupgrade/pgautoupgrade:latest + command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF" + restart: unless-stopped + environment: + POSTGRES_HOST_AUTH_METHOD: "trust" diff --git a/.ci/docker_build b/.ci/docker_build new file mode 100755 index 0000000000..324c7e996e --- /dev/null +++ b/.ci/docker_build @@ -0,0 +1,39 @@ +#!/bin/bash + +# This script only needs to run on the main Redash repo + +if [ "${GITHUB_REPOSITORY}" != "getredash/redash" ]; then + echo "Skipping image build for Docker Hub, as this isn't the main Redash repository" + exit 0 +fi + +if [ "${GITHUB_REF_NAME}" != "master" ] && [ "${GITHUB_REF_NAME}" != "preview-image" ]; then + echo "Skipping image build for Docker Hub, as this isn't the 'master' nor 'preview-image' branch" + exit 0 +fi + +if [ "x${DOCKER_USER}" = "x" ] || [ "x${DOCKER_PASS}" = "x" ]; then + echo "Skipping image build for Docker Hub, as the login details aren't available" + exit 0 +fi + +set -e +VERSION=$(jq -r .version package.json) +VERSION_TAG="$VERSION.b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}" + +export DOCKER_BUILDKIT=1 +export COMPOSE_DOCKER_CLI_BUILD=1 + +docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" + +DOCKERHUB_REPO="redash/redash" +DOCKER_TAGS="-t redash/redash:preview -t redash/preview:${VERSION_TAG}" + +# Build the docker container +docker build --build-arg install_groups="main,all_ds,dev" ${DOCKER_TAGS} . + +# Push the container to the preview build locations +docker push "${DOCKERHUB_REPO}:preview" +docker push "redash/preview:${VERSION_TAG}" + +echo "Built: ${VERSION_TAG}" diff --git a/.ci/pack b/.ci/pack new file mode 100755 index 0000000000..16223c5a9b --- /dev/null +++ b/.ci/pack @@ -0,0 +1,9 @@ +#!/bin/bash +NAME=redash +VERSION=$(jq -r .version package.json) +FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM +FILENAME=$NAME.$FULL_VERSION.tar.gz + +mkdir -p /tmp/artifacts/ + +tar -zcv -f /tmp/artifacts/$FILENAME --exclude=".git" --exclude="optipng*" --exclude="cypress" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" * diff --git a/.ci/update_version b/.ci/update_version new file mode 100755 index 0000000000..53b537208c --- /dev/null +++ b/.ci/update_version @@ -0,0 +1,6 @@ +#!/bin/bash +VERSION=$(jq -r .version package.json) +FULL_VERSION=${VERSION}+b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER} + +sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py +sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f62f01e47..e8098f5b0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,29 +3,12 @@ on: push: branches: - master - tags: - - '*' pull_request_target: branches: - master env: - CYPRESS_COVERAGE: "true" NODE_VERSION: 18 YARN_VERSION: 1.22.22 - REDASH_COOKIE_SECRET: 2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF - REDASH_SECRET_KEY: 2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF - COMPOSE_DOCKER_CLI_BUILD: 1 - DOCKER_BUILDKIT: 1 - FRONTEND_BUILD_MODE: 1 - INSTALL_GROUPS: main,all_ds,dev - PERCY_BRANCH: ${{github.head_ref || github.ref_name}} - PERCY_COMMIT: ${{github.sha}} - PERCY_PULL_REQUEST: ${{github.event.number}} - COMMIT_INFO_BRANCH: ${{github.head_ref || github.ref_name}} - COMMIT_INFO_MESSAGE: ${{github.event.head_commit.message}} - COMMIT_INFO_AUTHOR: ${{github.event.pull_request.user.login}} - COMMIT_INFO_SHA: ${{github.sha}} - COMMIT_INFO_REMOTE: ${{github.server_url}}/${{github.repository}} jobs: backend-lint: runs-on: ubuntu-22.04 @@ -40,7 +23,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.8' - - run: sudo pip install black==24.3.0 ruff==0.1.9 + - run: sudo pip install black==23.1.0 ruff==0.0.287 - run: ruff check . - run: black --check . @@ -48,7 +31,10 @@ jobs: runs-on: ubuntu-22.04 needs: backend-lint env: - FRONTEND_BUILD_MODE: 0 + COMPOSE_FILE: .ci/compose.ci.yaml + COMPOSE_PROJECT_NAME: redash + COMPOSE_DOCKER_CLI_BUILD: 1 + DOCKER_BUILDKIT: 1 steps: - if: github.event.pull_request.mergeable == 'false' name: Exit if PR is not mergeable @@ -60,16 +46,15 @@ jobs: - name: Build Docker Images run: | set -x - touch .env - docker compose build + docker compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true docker compose up -d sleep 10 - name: Create Test Database - run: docker compose run --rm postgres psql -h postgres -U postgres -c "create database tests;" + run: docker compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;" - name: List Enabled Query Runners - run: docker compose run --rm server manage ds list_types + run: docker compose -p redash run --rm redash manage ds list_types - name: Run Tests - run: docker compose run --name tests server tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/ + run: docker compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/ - name: Copy Test Results run: | mkdir -p /tmp/test-results/unit-tests @@ -77,17 +62,15 @@ jobs: docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - name: Store Test Results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: - name: backend-test-results + name: test-results path: /tmp/test-results - name: Store Coverage Results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: - name: backend-coverage + name: coverage path: coverage.xml frontend-lint: @@ -107,14 +90,13 @@ jobs: - name: Install Dependencies run: | npm install --global --force yarn@$YARN_VERSION - yarn cache clean - yarn --frozen-lockfile --network-concurrency 1 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - name: Run Lint run: yarn lint:ci - name: Store Test Results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: - name: frontend-test-results + name: test-results path: /tmp/test-results frontend-unit-tests: @@ -135,24 +117,21 @@ jobs: - name: Install Dependencies run: | npm install --global --force yarn@$YARN_VERSION - yarn cache clean - yarn --frozen-lockfile --network-concurrency 1 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - name: Run App Tests run: yarn test - name: Run Visualizations Tests - run: | - cd viz-lib - yarn test + run: cd viz-lib && yarn test - run: yarn lint frontend-e2e-tests: runs-on: ubuntu-22.04 needs: frontend-lint env: + COMPOSE_FILE: .ci/compose.cypress.yaml + COMPOSE_PROJECT_NAME: cypress CYPRESS_INSTALL_BINARY: 0 PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1 - INSTALL_GROUPS: main - COMPOSE_PROFILES: e2e PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} @@ -168,16 +147,17 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} cache: 'yarn' + - name: Enable Code Coverage Report For Master Branch + if: endsWith(github.ref, '/master') + run: | + echo "CODE_COVERAGE=true" >> "$GITHUB_ENV" - name: Install Dependencies run: | npm install --global --force yarn@$YARN_VERSION - yarn cache clean - yarn --frozen-lockfile --network-concurrency 1 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 - name: Setup Redash Server run: | set -x - touch .env - yarn build yarn cypress build yarn cypress start -- --skip-db-seed docker compose run cypress yarn cypress db-seed @@ -189,12 +169,7 @@ jobs: - name: Copy Code Coverage Results run: docker cp cypress:/usr/src/app/coverage ./coverage || true - name: Store Coverage Results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: - name: frontend-coverage + name: coverage path: coverage - - uses: actions/upload-artifact@v4 - with: - name: frontend - path: client/dist - retention-days: 1 diff --git a/.github/workflows/preview-image.yml b/.github/workflows/preview-image.yml index 9a691fb6f4..ee81a6911f 100644 --- a/.github/workflows/preview-image.yml +++ b/.github/workflows/preview-image.yml @@ -1,20 +1,15 @@ name: Preview Image on: - workflow_run: - workflows: - - Tests - types: - - completed - branches: - - master + push: + tags: + - '*-dev' env: - DOCKER_REPO: redash + NODE_VERSION: 18 jobs: build-skip-check: runs-on: ubuntu-22.04 - if: ${{ github.event.workflow_run.conclusion == 'success' }} outputs: skip: ${{ steps.skip-check.outputs.skip }} steps: @@ -37,118 +32,56 @@ jobs: runs-on: ubuntu-22.04 needs: - build-skip-check - outputs: - version: ${{ steps.version.outputs.VERSION_TAG }} - repo: ${{ steps.version.outputs.DOCKER_REPO }} if: needs.build-skip-check.outputs.skip == 'false' - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 steps: - uses: actions/checkout@v4 with: fetch-depth: 1 ref: ${{ github.event.push.after }} - - uses: dawidd6/action-download-artifact@v3 + + - uses: actions/setup-node@v4 with: - name: frontend - workflow: ci.yml - github_token: ${{ secrets.GITHUB_TOKEN }} - run_id: ${{ github.event.workflow_run.id }} - path: client/dist + node-version: ${{ env.NODE_VERSION }} + cache: 'yarn' + + - name: Install Dependencies + run: | + npm install --global --force yarn@1.22.22 + yarn cache clean && yarn --frozen-lockfile --network-concurrency 1 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ vars.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} + - name: Set version id: version run: | set -x - VERSION=$(jq -r .version package.json) - FULL_VERSION=${VERSION}-b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER} - sed -ri "s/^__version__ = ([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py - sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json - echo "VERSION_TAG=$FULL_VERSION" >> "$GITHUB_OUTPUT" - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - echo "SCOPE=${platform//\//-}" >> $GITHUB_ENV - if [[ "${{ vars.DOCKER_REPO }}" != "" ]]; then - echo "DOCKER_REPO=${{ vars.DOCKER_REPO }}" >> $GITHUB_ENV - echo "DOCKER_REPO=${{ vars.DOCKER_REPO }}" >> $GITHUB_OUTPUT - else - echo "DOCKER_REPO=${DOCKER_REPO}" >> $GITHUB_ENV - echo "DOCKER_REPO=${DOCKER_REPO}" >> $GITHUB_OUTPUT - fi + .ci/update_version + VERSION_TAG=$(jq -r .version package.json) + echo "VERSION_TAG=$VERSION_TAG" >> "$GITHUB_OUTPUT" + - name: Build and push preview image to Docker Hub - uses: docker/build-push-action@v5 - id: build + uses: docker/build-push-action@v4 with: push: true + tags: | + redash/redash:preview + redash/preview:${{ steps.version.outputs.VERSION_TAG }} context: . - cache-from: type=gha,scope=${{ env.SCOPE }} - cache-to: type=gha,mode=max,scope=${{ env.SCOPE }} - platforms: ${{ matrix.platform }} - outputs: type=image,name=${{ env.DOCKER_REPO }}/redash,push-by-digest=true,name-canonical=true,push=true build-args: | - FRONTEND_BUILD_MODE=1 + test_all_deps=true + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 env: DOCKER_CONTENT_TRUST: true - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - publish-docker-manifest: - runs-on: ubuntu-22.04 - needs: - - build-skip-check - - build-docker-image - if: needs.build-skip-check.outputs.skip == 'false' - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - pattern: digests-* - path: /tmp/digests - merge-multiple: true - - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ needs.build-docker-image.outputs.repo }}/redash - tags: preview - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASS }} - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ needs.build-docker-image.outputs.repo }}/redash@sha256:%s ' *) - - name: Inspect image - run: | - REDASH_IMAGE="${{ needs.build-docker-image.outputs.repo }}/redash:${{ steps.meta.outputs.version }}" - docker buildx imagetools inspect $REDASH_IMAGE - - name: Push image ${{ needs.build-docker-image.outputs.repo }}/preview image - run: | - REDASH_IMAGE="${{ needs.build-docker-image.outputs.repo }}/redash:preview" - PREVIEW_IMAGE="${{ needs.build-docker-image.outputs.repo }}/preview:${{ needs.build-docker-image.outputs.version }}" - docker buildx imagetools create --tag $PREVIEW_IMAGE $REDASH_IMAGE + - name: "Failure: output container logs to console" + if: failure() + run: docker compose logs diff --git a/.nvmrc b/.nvmrc index 53d0020fde..3f430af82b 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v16.20.1 +v18 diff --git a/Dockerfile b/Dockerfile index c4497d1335..b7d5ed23d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,30 @@ -# Controls whether to build the frontend assets -ARG FRONTEND_BUILD_MODE=0 +FROM node:18-bookworm as frontend-builder -# MODE 0: create empty files. useful for backend tests -FROM alpine:3.19 as frontend-builder-0 -RUN \ - mkdir -p /frontend/client/dist && \ - touch /frontend/client/dist/multi_org.html && \ - touch /frontend/client/dist/index.html +RUN npm install --global --force yarn@1.22.22 -# MODE 1: copy static frontend from host, useful for CI to ignore building static content multiple times -FROM alpine:3.19 as frontend-builder-1 -COPY client/dist /frontend/client/dist +# Controls whether to build the frontend assets +ARG skip_frontend_build -# MODE 2: build static content in docker, can be used for a local development -FROM node:18-bookworm as frontend-builder-2 -RUN npm install --global --force yarn@1.22.22 ENV CYPRESS_INSTALL_BINARY=0 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1 + RUN useradd -m -d /frontend redash USER redash + WORKDIR /frontend COPY --chown=redash package.json yarn.lock .yarnrc /frontend/ COPY --chown=redash viz-lib /frontend/viz-lib COPY --chown=redash scripts /frontend/scripts -RUN yarn --frozen-lockfile --network-concurrency 1; +# Controls whether to instrument code for coverage information +ARG code_coverage +ENV BABEL_ENV=${code_coverage:+test} + +RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn --frozen-lockfile --network-concurrency 1; fi + COPY --chown=redash client /frontend/client COPY --chown=redash webpack.config.js /frontend/ -RUN yarn build - -FROM frontend-builder-${FRONTEND_BUILD_MODE} as frontend-builder +RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn build; else mkdir -p /frontend/client/dist && touch /frontend/client/dist/multi_org.html && touch /frontend/client/dist/index.html; fi FROM python:3.8-slim-bookworm @@ -66,18 +61,17 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN \ - curl https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list && \ - curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg && \ - apt update && \ - ACCEPT_EULA=Y apt install -y --no-install-recommends msodbcsql18 && \ - apt clean && \ - rm -rf /var/lib/apt/lists/* ARG TARGETPLATFORM ARG databricks_odbc_driver_url=https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.6.26/SimbaSparkODBC-2.6.26.1045-Debian-64bit.zip RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \ - curl "$databricks_odbc_driver_url" --location --output /tmp/simba_odbc.zip \ + curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \ + && curl https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list \ + && apt-get update \ + && ACCEPT_EULA=Y apt-get install -y --no-install-recommends msodbcsql18 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && curl "$databricks_odbc_driver_url" --location --output /tmp/simba_odbc.zip \ && chmod 600 /tmp/simba_odbc.zip \ && unzip /tmp/simba_odbc.zip -d /tmp/simba \ && dpkg -i /tmp/simba/*.deb \ @@ -97,8 +91,8 @@ COPY pyproject.toml poetry.lock ./ ARG POETRY_OPTIONS="--no-root --no-interaction --no-ansi" # for LDAP authentication, install with `ldap3` group # disabled by default due to GPL license conflict -ARG INSTALL_GROUPS="main,all_ds,dev" -RUN /etc/poetry/bin/poetry install --only $INSTALL_GROUPS $POETRY_OPTIONS +ARG install_groups="main,all_ds,dev" +RUN /etc/poetry/bin/poetry install --only $install_groups $POETRY_OPTIONS COPY --chown=redash . /app COPY --from=frontend-builder --chown=redash /frontend/client/dist /app/client/dist diff --git a/Makefile b/Makefile index c6b0363543..404bfc10e1 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,10 @@ -.PHONY: compose_build up test_db create_database create_db clean clean-all down tests lint backend-unit-tests frontend-unit-tests pydeps test build watch start redis-cli bash - -export COMPOSE_DOCKER_CLI_BUILD=1 -export DOCKER_BUILDKIT=1 -export COMPOSE_PROFILES=local +.PHONY: compose_build up test_db create_database clean clean-all down tests lint backend-unit-tests frontend-unit-tests test build watch start redis-cli bash compose_build: .env - docker compose build + COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose build up: - docker compose up -d redis postgres - docker compose exec -u postgres postgres psql postgres --csv \ - -1tqc "SELECT table_name FROM information_schema.tables WHERE table_name = 'organizations'" 2> /dev/null \ - | grep -q "organizations" || make create_database - docker compose up -d --build + COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose up -d --build test_db: @for i in `seq 1 5`; do \ @@ -21,11 +13,9 @@ test_db: done docker compose exec postgres sh -c 'psql -U postgres -c "drop database if exists tests;" && psql -U postgres -c "create database tests;"' -create_db: .env +create_database: .env docker compose run server create_db -create_database: create_db - clean: docker compose down docker compose --project-name cypress down @@ -54,12 +44,6 @@ env: .env format: pre-commit run --all-files -pydeps: - pip3 install wheel - pip3 install --upgrade black ruff launchpadlib pip setuptools - pip3 install poetry - poetry install --only main,all_ds,dev - tests: docker compose run server tests diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint index 48a8621a92..5e777c10ed 100755 --- a/bin/docker-entrypoint +++ b/bin/docker-entrypoint @@ -1,48 +1,25 @@ #!/bin/bash set -e -if [ -z $REDASH_REDIS_URL ]; then - export REDASH_REDIS_URL=redis://:${REDASH_REDIS_PASSWORD}@${REDASH_REDIS_HOSTNAME}:${REDASH_REDIS_PORT}/${REDASH_REDIS_NAME} -fi - -if [ -z $REDASH_DATABASE_URL ]; then - export REDASH_DATABASE_URL=postgresql://${REDASH_DATABASE_USER}:${REDASH_DATABASE_PASSWORD}@${REDASH_DATABASE_HOSTNAME}:${REDASH_DATABASE_PORT}/${REDASH_DATABASE_NAME} -fi - scheduler() { echo "Starting RQ scheduler..." - case $REDASH_PRODUCTION in - true) - echo "Starting RQ scheduler in production mode" - exec ./manage.py rq scheduler - ;; - *) - echo "Starting RQ scheduler in dev mode" - exec watchmedo auto-restart \ - --directory=./redash/ \ - --pattern=*.py \ - --recursive -- ./manage.py rq scheduler $QUEUES - ;; - esac + exec /app/manage.py rq scheduler +} + +dev_scheduler() { + echo "Starting dev RQ scheduler..." + + exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq scheduler } worker() { + echo "Starting RQ worker..." + export WORKERS_COUNT=${WORKERS_COUNT:-2} export QUEUES=${QUEUES:-} - case $REDASH_PRODUCTION in - true) - echo "Starting RQ worker in production mode" - exec supervisord -c worker.conf - ;; - *) - echo "Starting RQ worker in dev mode" - exec watchmedo auto-restart \ - --directory=./redash/ \ - --pattern=*.py \ - --recursive -- ./manage.py rq worker $QUEUES - ;; - esac + + exec supervisord -c worker.conf } workers_healthcheck() { @@ -58,63 +35,22 @@ workers_healthcheck() { fi } +dev_worker() { + echo "Starting dev RQ worker..." + + exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq worker $QUEUES +} + server() { # Recycle gunicorn workers every n-th request. See http://docs.gunicorn.org/en/stable/settings.html#max-requests for more details. - case $REDASH_PRODUCTION in - true) - echo "Starting Redash Server in production mode" - MAX_REQUESTS=${MAX_REQUESTS:-1000} - MAX_REQUESTS_JITTER=${MAX_REQUESTS_JITTER:-100} - TIMEOUT=${REDASH_GUNICORN_TIMEOUT:-60} - exec /usr/local/bin/gunicorn \ - -b 0.0.0.0:5000 \ - --name redash \ - -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app \ - --max-requests $MAX_REQUESTS \ - --max-requests-jitter $MAX_REQUESTS_JITTER \ - --timeout $TIMEOUT - ;; - *) - echo "Starting Redash Server in a dev mode" - export FLASK_DEBUG=1 - exec /app/manage.py runserver --debugger --reload -h 0.0.0.0 - ;; - esac + MAX_REQUESTS=${MAX_REQUESTS:-1000} + MAX_REQUESTS_JITTER=${MAX_REQUESTS_JITTER:-100} + TIMEOUT=${REDASH_GUNICORN_TIMEOUT:-60} + exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app --max-requests $MAX_REQUESTS --max-requests-jitter $MAX_REQUESTS_JITTER --timeout $TIMEOUT } create_db() { - REDASH_DATABASE_MIGRATE_TIMEOUT=${REDASH_DATABASE_UPGRADE_TIMEOUT:-600} - REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS=${REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS:-5} - REDASH_DATABASE_MIGRATE_RETRY_WAIT=${REDASH_DATABASE_MIGRATE_RETRY_WAIT:-10} - ATTEMPTS=1 - while ((ATTEMPTS <= REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS)); do - echo "Creating or updating Redash database, attempt ${ATTEMPTS} of ${REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS}" - ATTEMPTS=$((ATTEMPTS+1)) - timeout $REDASH_DATABASE_MIGRATE_TIMEOUT /app/manage.py database create_tables - timeout $REDASH_DATABASE_MIGRATE_TIMEOUT /app/manage.py db upgrade - STATUS=$(timeout $REDASH_DATABASE_MIGRATE_TIMEOUT /app/manage.py status 2>&1) - RETCODE=$? - case "$RETCODE" in - 0) - exit 0 - ;; - 124) - echo "Status command timed out after ${REDASH_DATABASE_MIGRATE_TIMEOUT} seconds." - ;; - esac - case "$STATUS" in - *sqlalchemy.exc.OperationalError*) - echo "Database not yet functional, waiting." - ;; - *sqlalchemy.exc.ProgrammingError*) - echo "Database does not appear to be installed." - ;; - esac - echo "Waiting ${REDASH_DATABASE_MIGRATE_RETRY_WAIT} seconds before retrying." - sleep ${REDASH_DATABASE_MIGRATE_RETRY_WAIT} - done - echo "Reached ${REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS} attempts, giving up." - exit 1 + exec /app/manage.py database create_tables } help() { @@ -125,16 +61,21 @@ help() { echo "server -- start Redash server (with gunicorn)" echo "worker -- start a single RQ worker" + echo "dev_worker -- start a single RQ worker with code reloading" echo "scheduler -- start an rq-scheduler instance" + echo "dev_scheduler -- start an rq-scheduler instance with code reloading" echo "" echo "shell -- open shell" + echo "dev_server -- start Flask development server with debugger and auto reload" echo "debug -- start Flask development server with remote debugger via ptvsd" - echo "create_db -- create database tables and run migrations" + echo "create_db -- create database tables" echo "manage -- CLI to manage redash" echo "tests -- run tests" } tests() { + export REDASH_DATABASE_URL="postgresql://postgres@postgres/tests" + if [ $# -eq 0 ]; then TEST_ARGS=tests/ else @@ -160,10 +101,22 @@ case "$1" in shift scheduler ;; + dev_scheduler) + shift + dev_scheduler + ;; + dev_worker) + shift + dev_worker + ;; celery_healthcheck) shift echo "DEPRECATED: Celery has been replaced with RQ and now performs healthchecks autonomously as part of the 'worker' entrypoint." ;; + dev_server) + export FLASK_DEBUG=1 + exec /app/manage.py runserver --debugger --reload -h 0.0.0.0 + ;; debug) export FLASK_DEBUG=1 export REMOTE_DEBUG=1 diff --git a/client/app/components/dashboards/dashboard-widget/VisualizationWidget.jsx b/client/app/components/dashboards/dashboard-widget/VisualizationWidget.jsx index 58657e4c6d..9a021cc8bd 100644 --- a/client/app/components/dashboards/dashboard-widget/VisualizationWidget.jsx +++ b/client/app/components/dashboards/dashboard-widget/VisualizationWidget.jsx @@ -19,7 +19,6 @@ import PlainButton from "@/components/PlainButton"; import ExpandedWidgetDialog from "@/components/dashboards/ExpandedWidgetDialog"; import EditParameterMappingsDialog from "@/components/dashboards/EditParameterMappingsDialog"; import VisualizationRenderer from "@/components/visualizations/VisualizationRenderer"; -import { ExecutionStatus } from "@/services/query-result"; import Widget from "./Widget"; @@ -279,7 +278,7 @@ class VisualizationWidget extends React.Component { const widgetQueryResult = widget.getQueryResult(); const widgetStatus = widgetQueryResult && widgetQueryResult.getStatus(); switch (widgetStatus) { - case ExecutionStatus.FAILED: + case "failed": return (
{widgetQueryResult.getError() && ( @@ -289,7 +288,7 @@ class VisualizationWidget extends React.Component { )}
); - case ExecutionStatus.FINISHED: + case "done": return (
- {item.description ? ( - - -