feat: docker-compose to work off repo Dockerfile (#27434)

UPDATING.md

@@ -28,6 +28,11 @@ assists people when migrating to a new version.
 
 - [26450](https://github.com/apache/superset/pull/26450): Deprecates the `KV_STORE` feature flag and its related assets such as the API endpoint and `keyvalue` table. The main dependency of this feature is the `SHARE_QUERIES_VIA_KV_STORE` feature flag which allows sharing SQL Lab queries without the necessity of saving the query. Our intention is to use the permalink feature to implement this use case before 5.0 and that's why we are deprecating the feature flag now.
 
+- [27434](https://github.com/apache/superset/pull/27434/files): DO NOT USE our docker-compose.*
+  files for production use cases! While we never really supported
+  or should have tried to support docker-compose for production use cases, we now actively
+  have taken a stance against supporting it. See the PR for details.
+
 ### Breaking Changes
 
 - [27130](https://github.com/apache/superset/pull/27130): Fixes the DELETE `/database/{id}/ssh_tunnel/`` endpoint to now correctly accept a database ID as a parameter, rather than an SSH tunnel ID.

docker-compose-image-tag.yml

@@ -0,0 +1,101 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest}
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis
+x-superset-volumes:
+  &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ./docker:/app/docker
+  - superset_home:/app/superset_home
+
+version: "3.7"
+services:
+  redis:
+    image: redis:7
+    container_name: superset_cache
+    restart: unless-stopped
+    volumes:
+      - redis:/data
+
+  db:
+    env_file: docker/.env
+    image: postgres:15
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+      - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
+
+  superset:
+    env_file: docker/.env
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
+    ports:
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  superset-init:
+    image: *superset-image
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: docker/.env
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+    healthcheck:
+      disable: true
+
+  superset-worker:
+    image: *superset-image
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: docker/.env
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "celery -A superset.tasks.celery_app:app inspect ping -d celery@$$HOSTNAME",
+        ]
+
+  superset-worker-beat:
+    image: *superset-image
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: docker/.env
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+    healthcheck:
+      disable: true
+
+volumes:
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis:
+    external: false

docker-compose-non-dev.yml

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest}
 x-superset-depends-on: &superset-depends-on
   - db
   - redis
@@ -23,7 +22,13 @@ x-superset-volumes:
   - ./docker:/app/docker
   - superset_home:/app/superset_home
 
-version: "3.7"
+x-common-build: &common-build
+  context: .
+  target: dev
+  cache_from:
+    - apache/superset-cache:3.9-slim-bookworm
+
+version: "4.0"
 services:
   redis:
     image: redis:7
@@ -33,7 +38,7 @@ services:
       - redis:/data
 
   db:
-    env_file: docker/.env-non-dev
+    env_file: docker/.env
     image: postgres:15
     container_name: superset_db
     restart: unless-stopped
@@ -42,8 +47,9 @@ services:
       - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
 
   superset:
-    env_file: docker/.env-non-dev
-    image: *superset-image
+    env_file: docker/.env
+    build:
+      <<: *common-build
     container_name: superset_app
     command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
     user: "root"
@@ -54,21 +60,23 @@ services:
     volumes: *superset-volumes
 
   superset-init:
-    image: *superset-image
     container_name: superset_init
+    build:
+      <<: *common-build
     command: ["/app/docker/docker-init.sh"]
-    env_file: docker/.env-non-dev
+    env_file: docker/.env
     depends_on: *superset-depends-on
     user: "root"
     volumes: *superset-volumes
     healthcheck:
       disable: true
 
   superset-worker:
-    image: *superset-image
+    build:
+      <<: *common-build
     container_name: superset_worker
     command: ["/app/docker/docker-bootstrap.sh", "worker"]
-    env_file: docker/.env-non-dev
+    env_file: docker/.env
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: "root"
@@ -81,10 +89,11 @@ services:
         ]
 
   superset-worker-beat:
-    image: *superset-image
+    build:
+      <<: *common-build
     container_name: superset_worker_beat
     command: ["/app/docker/docker-bootstrap.sh", "beat"]
-    env_file: docker/.env-non-dev
+    env_file: docker/.env
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: "root"

docker-compose.yml

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-master-dev}
 x-superset-user: &superset-user root
 x-superset-depends-on: &superset-depends-on
   - db
@@ -27,7 +26,13 @@ x-superset-volumes: &superset-volumes
   - superset_home:/app/superset_home
   - ./tests:/app/tests
 
-version: "3.7"
+x-common-build: &common-build
+  context: .
+  target: dev
+  cache_from:
+    - apache/superset-cache:3.9-slim-bookworm
+
+version: "4.0"
 services:
   nginx:
     image: nginx:latest
@@ -61,7 +66,8 @@ services:
 
   superset:
     env_file: docker/.env
-    image: *superset-image
+    build:
+      <<: *common-build
     container_name: superset_app
     command: ["/app/docker/docker-bootstrap.sh", "app"]
     restart: unless-stopped
@@ -106,7 +112,8 @@ services:
       - REDIS_SSL=false
 
   superset-init:
-    image: *superset-image
+    build:
+      <<: *common-build
     container_name: superset_init
     command: ["/app/docker/docker-init.sh"]
     env_file: docker/.env
@@ -120,16 +127,21 @@ services:
 
   superset-node:
     image: node:16
+    environment:
+      # set this to false if you have perf issues running the npm i; npm run dev in-docker
+      # if you do so, you have to run this manually on the host, which should perform better!
+      BUILD_SUPERSET_FRONTEND_IN_DOCKER: ${BUILD_SUPERSET_FRONTEND_IN_DOCKER:-true}
+      SCARF_ANALYTICS: "${SCARF_ANALYTICS}"
+      PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: ${BUILD_SUPERSET_FRONTEND_IN_DOCKER:-false}
     container_name: superset_node
     command: ["/app/docker/docker-frontend.sh"]
     env_file: docker/.env
     depends_on: *superset-depends-on
-    environment:
-      SCARF_ANALYTICS: "${SCARF_ANALYTICS}"
     volumes: *superset-volumes
 
   superset-worker:
-    image: *superset-image
+    build:
+      <<: *common-build
     container_name: superset_worker
     command: ["/app/docker/docker-bootstrap.sh", "worker"]
     env_file: docker/.env
@@ -146,7 +158,8 @@ services:
     # mem_reservation: 128M
 
   superset-worker-beat:
-    image: *superset-image
+    build:
+      <<: *common-build
     container_name: superset_worker_beat
     command: ["/app/docker/docker-bootstrap.sh", "beat"]
     env_file: docker/.env
@@ -158,7 +171,8 @@ services:
       disable: true
 
   superset-tests-worker:
-    image: *superset-image
+    build:
+      <<: *common-build
     container_name: superset_tests_worker
     command: ["/app/docker/docker-bootstrap.sh", "worker"]
     env_file: docker/.env

docker/.env

@@ -51,3 +51,5 @@ SUPERSET_LOAD_EXAMPLES=yes
 CYPRESS_CONFIG=false
 SUPERSET_PORT=8088
 MAPBOX_API_KEY=''
+
+SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET

docker/docker-frontend.sh

@@ -19,11 +19,17 @@ set -e
 
 # Packages needed for puppeteer:
 apt update
-apt install -y chromium
+if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = "false" ]; then
+    apt install -y chromium
+fi
 
-cd /app/superset-frontend
-npm install -f --no-optional --global webpack webpack-cli
-npm install -f --no-optional
+if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
+    cd /app/superset-frontend
+    npm install -f --no-optional --global webpack webpack-cli
+    npm install -f --no-optional
 
-echo "Running frontend"
-npm run dev
+    echo "Running frontend"
+    npm run dev
+else
+    echo "Skipping frontend build steps - YOU RUN IT MANUALLY ON THE HOST!"
+fi

docs/docs/frequently-asked-questions.mdx

@@ -89,7 +89,7 @@ SUPERSET_WEBSERVER_TIMEOUT = 60
 ### Why is the map not visible in the geospatial visualization?
 
 You need to register a free account at [Mapbox.com](https://www.mapbox.com), obtain an API key, and add it
-to **.env** and **.env-non-dev** at the key MAPBOX_API_KEY:
+to **.env** at the key MAPBOX_API_KEY:
 
 ```
 MAPBOX_API_KEY = "longstringofalphanumer1c"