From bd115e7f5fd5127c081f17fcfec2cad797407454 Mon Sep 17 00:00:00 2001
From: Andrii Chubatiuk <andrew.chubatiuk@gmail.com>
Date: Mon, 6 May 2024 16:14:13 +0300
Subject: [PATCH 01/12] Remove deprecated advocate package (#6944)

---
 poetry.lock                      | 77 +-------------------------------
 pyproject.toml                   |  1 -
 redash/query_runner/__init__.py  | 15 ++-----
 redash/query_runner/csv.py       | 10 +----
 redash/query_runner/excel.py     | 10 +----
 redash/settings/__init__.py      |  3 --
 redash/utils/requests_session.py | 18 --------
 tests/query_runner/test_http.py  | 21 ++++-----
 8 files changed, 18 insertions(+), 137 deletions(-)
 delete mode 100644 redash/utils/requests_session.py

diff --git a/poetry.lock b/poetry.lock
index a3ad412c46..6bec3f2ed1 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -17,26 +17,6 @@ PyJWT = ">=1.0.0,<3"
 python-dateutil = ">=2.1.0,<3"
 requests = ">=2.0.0,<3"
 
-[[package]]
-name = "advocate"
-version = "1.0.0"
-description = "A wrapper around the requests library for safely making HTTP requests on behalf of a third party"
-optional = false
-python-versions = "*"
-files = [
-    {file = "advocate-1.0.0-py2.py3-none-any.whl", hash = "sha256:e8b340e49fadc0e416fbc9e81ef52d74858ccad16357dabde6cf9d99a7407d70"},
-    {file = "advocate-1.0.0.tar.gz", hash = "sha256:1bf1170e41334279996580329c594e017540ab0eaf7a152323e743f0a85a353d"},
-]
-
-[package.dependencies]
-ndg-httpsclient = "*"
-netifaces = ">=0.10.5"
-pyasn1 = "*"
-pyopenssl = "*"
-requests = ">=2.18.0,<3.0"
-six = "*"
-urllib3 = ">=1.22,<2.0"
-
 [[package]]
 name = "alembic"
 version = "1.13.1"
@@ -2408,61 +2388,6 @@ files = [
     {file = "mysqlclient-2.1.1.tar.gz", hash = "sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782"},
 ]
 
-[[package]]
-name = "ndg-httpsclient"
-version = "0.5.1"
-description = "Provides enhanced HTTPS support for httplib and urllib2 using PyOpenSSL"
-optional = false
-python-versions = ">=2.7,<3.0.dev0 || >=3.4.dev0"
-files = [
-    {file = "ndg_httpsclient-0.5.1-py2-none-any.whl", hash = "sha256:d2c7225f6a1c6cf698af4ebc962da70178a99bcde24ee6d1961c4f3338130d57"},
-    {file = "ndg_httpsclient-0.5.1-py3-none-any.whl", hash = "sha256:dd174c11d971b6244a891f7be2b32ca9853d3797a72edb34fa5d7b07d8fff7d4"},
-    {file = "ndg_httpsclient-0.5.1.tar.gz", hash = "sha256:d72faed0376ab039736c2ba12e30695e2788c4aa569c9c3e3d72131de2592210"},
-]
-
-[package.dependencies]
-pyasn1 = ">=0.1.1"
-PyOpenSSL = "*"
-
-[[package]]
-name = "netifaces"
-version = "0.11.0"
-description = "Portable network interface information."
-optional = false
-python-versions = "*"
-files = [
-    {file = "netifaces-0.11.0-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eb4813b77d5df99903af4757ce980a98c4d702bbcb81f32a0b305a1537bdf0b1"},
-    {file = "netifaces-0.11.0-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:5f9ca13babe4d845e400921973f6165a4c2f9f3379c7abfc7478160e25d196a4"},
-    {file = "netifaces-0.11.0-cp27-cp27m-win32.whl", hash = "sha256:7dbb71ea26d304e78ccccf6faccef71bb27ea35e259fb883cfd7fd7b4f17ecb1"},
-    {file = "netifaces-0.11.0-cp27-cp27m-win_amd64.whl", hash = "sha256:0f6133ac02521270d9f7c490f0c8c60638ff4aec8338efeff10a1b51506abe85"},
-    {file = "netifaces-0.11.0-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:08e3f102a59f9eaef70948340aeb6c89bd09734e0dca0f3b82720305729f63ea"},
-    {file = "netifaces-0.11.0-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c03fb2d4ef4e393f2e6ffc6376410a22a3544f164b336b3a355226653e5efd89"},
-    {file = "netifaces-0.11.0-cp34-cp34m-win32.whl", hash = "sha256:73ff21559675150d31deea8f1f8d7e9a9a7e4688732a94d71327082f517fc6b4"},
-    {file = "netifaces-0.11.0-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815eafdf8b8f2e61370afc6add6194bd5a7252ae44c667e96c4c1ecf418811e4"},
-    {file = "netifaces-0.11.0-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:50721858c935a76b83dd0dd1ab472cad0a3ef540a1408057624604002fcfb45b"},
-    {file = "netifaces-0.11.0-cp35-cp35m-win32.whl", hash = "sha256:c9a3a47cd3aaeb71e93e681d9816c56406ed755b9442e981b07e3618fb71d2ac"},
-    {file = "netifaces-0.11.0-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:aab1dbfdc55086c789f0eb37affccf47b895b98d490738b81f3b2360100426be"},
-    {file = "netifaces-0.11.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c37a1ca83825bc6f54dddf5277e9c65dec2f1b4d0ba44b8fd42bc30c91aa6ea1"},
-    {file = "netifaces-0.11.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:28f4bf3a1361ab3ed93c5ef360c8b7d4a4ae060176a3529e72e5e4ffc4afd8b0"},
-    {file = "netifaces-0.11.0-cp36-cp36m-win32.whl", hash = "sha256:2650beee182fed66617e18474b943e72e52f10a24dc8cac1db36c41ee9c041b7"},
-    {file = "netifaces-0.11.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cb925e1ca024d6f9b4f9b01d83215fd00fe69d095d0255ff3f64bffda74025c8"},
-    {file = "netifaces-0.11.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:84e4d2e6973eccc52778735befc01638498781ce0e39aa2044ccfd2385c03246"},
-    {file = "netifaces-0.11.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18917fbbdcb2d4f897153c5ddbb56b31fa6dd7c3fa9608b7e3c3a663df8206b5"},
-    {file = "netifaces-0.11.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:48324183af7f1bc44f5f197f3dad54a809ad1ef0c78baee2c88f16a5de02c4c9"},
-    {file = "netifaces-0.11.0-cp37-cp37m-win32.whl", hash = "sha256:8f7da24eab0d4184715d96208b38d373fd15c37b0dafb74756c638bd619ba150"},
-    {file = "netifaces-0.11.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2479bb4bb50968089a7c045f24d120f37026d7e802ec134c4490eae994c729b5"},
-    {file = "netifaces-0.11.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:3ecb3f37c31d5d51d2a4d935cfa81c9bc956687c6f5237021b36d6fdc2815b2c"},
-    {file = "netifaces-0.11.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:96c0fe9696398253f93482c84814f0e7290eee0bfec11563bd07d80d701280c3"},
-    {file = "netifaces-0.11.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c92ff9ac7c2282009fe0dcb67ee3cd17978cffbe0c8f4b471c00fe4325c9b4d4"},
-    {file = "netifaces-0.11.0-cp38-cp38-win32.whl", hash = "sha256:d07b01c51b0b6ceb0f09fc48ec58debd99d2c8430b09e56651addeaf5de48048"},
-    {file = "netifaces-0.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:469fc61034f3daf095e02f9f1bbac07927b826c76b745207287bc594884cfd05"},
-    {file = "netifaces-0.11.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5be83986100ed1fdfa78f11ccff9e4757297735ac17391b95e17e74335c2047d"},
-    {file = "netifaces-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54ff6624eb95b8a07e79aa8817288659af174e954cca24cdb0daeeddfc03c4ff"},
-    {file = "netifaces-0.11.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:841aa21110a20dc1621e3dd9f922c64ca64dd1eb213c47267a2c324d823f6c8f"},
-    {file = "netifaces-0.11.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e76c7f351e0444721e85f975ae92718e21c1f361bda946d60a214061de1f00a1"},
-    {file = "netifaces-0.11.0.tar.gz", hash = "sha256:043a79146eb2907edf439899f262b3dfe41717d34124298ed281139a8b93ca32"},
-]
-
 [[package]]
 name = "nodeenv"
 version = "1.8.0"
@@ -5300,4 +5225,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.11"
-content-hash = "e7985ee5c3ca3a4389b4e85fda033a9b3b867dbbe4b4a7fca8ea5c35fc401148"
+content-hash = "10482b8930a9e3c91c95dc79105f8bc35d9f4440e112f41fba61b74ed7ae0871"
diff --git a/pyproject.toml b/pyproject.toml
index e5cc871457..8684dc3e81 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,7 +23,6 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = ">=3.8,<3.11"
-advocate = "1.0.0"
 aniso8601 = "8.0.0"
 authlib = "0.15.5"
 backoff = "2.2.1"
diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py
index 10d6d6edf5..14a42cc3ca 100644
--- a/redash/query_runner/__init__.py
+++ b/redash/query_runner/__init__.py
@@ -3,17 +3,13 @@
 from contextlib import ExitStack
 from functools import wraps
 
+import requests
 import sqlparse
 from dateutil import parser
 from rq.timeouts import JobTimeoutException
 from sshtunnel import open_tunnel
 
 from redash import settings, utils
-from redash.utils.requests_session import (
-    UnacceptableAddressException,
-    requests_or_advocate,
-    requests_session,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -379,7 +375,7 @@ def get_response(self, url, auth=None, http_method="get", **kwargs):
         error = None
         response = None
         try:
-            response = requests_session.request(http_method, url, auth=auth, **kwargs)
+            response = requests.request(http_method, url, auth=auth, **kwargs)
             # Raise a requests HTTP exception with the appropriate reason
             # for 4xx and 5xx response status codes which is later caught
             # and passed back.
@@ -389,14 +385,11 @@ def get_response(self, url, auth=None, http_method="get", **kwargs):
             if response.status_code != 200:
                 error = "{} ({}).".format(self.response_error, response.status_code)
 
-        except requests_or_advocate.HTTPError as exc:
+        except requests.HTTPError as exc:
             logger.exception(exc)
             error = "Failed to execute query. "
             f"Return Code: {response.status_code} Reason: {response.text}"
-        except UnacceptableAddressException as exc:
-            logger.exception(exc)
-            error = "Can't query private addresses."
-        except requests_or_advocate.RequestException as exc:
+        except requests.RequestException as exc:
             # Catch all other requests exceptions and return the error.
             logger.exception(exc)
             error = str(exc)
diff --git a/redash/query_runner/csv.py b/redash/query_runner/csv.py
index 3d3cf61b9c..4c72f3c3fa 100644
--- a/redash/query_runner/csv.py
+++ b/redash/query_runner/csv.py
@@ -1,13 +1,10 @@
 import io
 import logging
 
+import requests
 import yaml
 
 from redash.query_runner import BaseQueryRunner, NotSupported, register
-from redash.utils.requests_session import (
-    UnacceptableAddressException,
-    requests_or_advocate,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -59,7 +56,7 @@ def run_query(self, query, user):
             pass
 
         try:
-            response = requests_or_advocate.get(url=path, headers={"User-agent": ua})
+            response = requests.get(url=path, headers={"User-agent": ua})
             workbook = pd.read_csv(io.BytesIO(response.content), sep=",", **args)
 
             df = workbook.copy()
@@ -99,9 +96,6 @@ def run_query(self, query, user):
         except KeyboardInterrupt:
             error = "Query cancelled by user."
             data = None
-        except UnacceptableAddressException:
-            error = "Can't query private addresses."
-            data = None
         except Exception as e:
             error = "Error reading {0}. {1}".format(path, str(e))
             data = None
diff --git a/redash/query_runner/excel.py b/redash/query_runner/excel.py
index 488632e022..5606149b31 100644
--- a/redash/query_runner/excel.py
+++ b/redash/query_runner/excel.py
@@ -1,12 +1,9 @@
 import logging
 
+import requests
 import yaml
 
 from redash.query_runner import BaseQueryRunner, NotSupported, register
-from redash.utils.requests_session import (
-    UnacceptableAddressException,
-    requests_or_advocate,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -57,7 +54,7 @@ def run_query(self, query, user):
             pass
 
         try:
-            response = requests_or_advocate.get(url=path, headers={"User-agent": ua})
+            response = requests.get(url=path, headers={"User-agent": ua})
             workbook = pd.read_excel(response.content, **args)
 
             df = workbook.copy()
@@ -97,9 +94,6 @@ def run_query(self, query, user):
         except KeyboardInterrupt:
             error = "Query cancelled by user."
             data = None
-        except UnacceptableAddressException:
-            error = "Can't query private addresses."
-            data = None
         except Exception as e:
             error = "Error reading {0}. {1}".format(path, str(e))
             data = None
diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py
index 75c438cff2..449f773a19 100644
--- a/redash/settings/__init__.py
+++ b/redash/settings/__init__.py
@@ -71,9 +71,6 @@
 # Whether file downloads are enforced or not.
 ENFORCE_FILE_SAVE = parse_boolean(os.environ.get("REDASH_ENFORCE_FILE_SAVE", "true"))
 
-# Whether api calls using the json query runner will block private addresses
-ENFORCE_PRIVATE_ADDRESS_BLOCK = parse_boolean(os.environ.get("REDASH_ENFORCE_PRIVATE_IP_BLOCK", "true"))
-
 # Whether to use secure cookies by default.
 COOKIES_SECURE = parse_boolean(os.environ.get("REDASH_COOKIES_SECURE", str(ENFORCE_HTTPS)))
 # Whether the session cookie is set to secure.
diff --git a/redash/utils/requests_session.py b/redash/utils/requests_session.py
deleted file mode 100644
index c562894181..0000000000
--- a/redash/utils/requests_session.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from advocate.exceptions import UnacceptableAddressException  # noqa: F401
-
-from redash import settings
-
-if settings.ENFORCE_PRIVATE_ADDRESS_BLOCK:
-    import advocate as requests_or_advocate
-else:
-    import requests as requests_or_advocate
-
-
-class ConfiguredSession(requests_or_advocate.Session):
-    def request(self, *args, **kwargs):
-        if not settings.REQUESTS_ALLOW_REDIRECTS:
-            kwargs.update({"allow_redirects": False})
-        return super().request(*args, **kwargs)
-
-
-requests_session = ConfiguredSession()
diff --git a/tests/query_runner/test_http.py b/tests/query_runner/test_http.py
index 1fa361cb92..d3e439ccb9 100644
--- a/tests/query_runner/test_http.py
+++ b/tests/query_runner/test_http.py
@@ -1,12 +1,9 @@
 from unittest import TestCase
 
 import mock
+import requests
 
 from redash.query_runner import BaseHTTPQueryRunner
-from redash.utils.requests_session import (
-    ConfiguredSession,
-    requests_or_advocate,
-)
 
 
 class RequiresAuthQueryRunner(BaseHTTPQueryRunner):
@@ -37,7 +34,7 @@ def test_get_auth_empty_requires_authentication(self):
         query_runner = RequiresAuthQueryRunner({})
         self.assertRaisesRegex(ValueError, "Username and Password required", query_runner.get_auth)
 
-    @mock.patch.object(ConfiguredSession, "request")
+    @mock.patch("requests.request")
     def test_get_response_success(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 200
@@ -51,7 +48,7 @@ def test_get_response_success(self, mock_get):
         self.assertEqual(response.status_code, 200)
         self.assertIsNone(error)
 
-    @mock.patch.object(ConfiguredSession, "request")
+    @mock.patch("requests.request")
     def test_get_response_success_custom_auth(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 200
@@ -66,7 +63,7 @@ def test_get_response_success_custom_auth(self, mock_get):
         self.assertEqual(response.status_code, 200)
         self.assertIsNone(error)
 
-    @mock.patch.object(ConfiguredSession, "request")
+    @mock.patch("requests.request")
     def test_get_response_failure(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 301
@@ -79,12 +76,12 @@ def test_get_response_failure(self, mock_get):
         mock_get.assert_called_once_with("get", url, auth=None)
         self.assertIn(query_runner.response_error, error)
 
-    @mock.patch.object(ConfiguredSession, "request")
+    @mock.patch("requests.request")
     def test_get_response_httperror_exception(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 500
         mock_response.text = "Server Error"
-        http_error = requests_or_advocate.HTTPError()
+        http_error = requests.HTTPError()
         mock_response.raise_for_status.side_effect = http_error
         mock_get.return_value = mock_response
 
@@ -95,13 +92,13 @@ def test_get_response_httperror_exception(self, mock_get):
         self.assertIsNotNone(error)
         self.assertIn("Failed to execute query", error)
 
-    @mock.patch.object(ConfiguredSession, "request")
+    @mock.patch("requests.request")
     def test_get_response_requests_exception(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 500
         mock_response.text = "Server Error"
         exception_message = "Some requests exception"
-        requests_exception = requests_or_advocate.RequestException(exception_message)
+        requests_exception = requests.RequestException(exception_message)
         mock_response.raise_for_status.side_effect = requests_exception
         mock_get.return_value = mock_response
 
@@ -112,7 +109,7 @@ def test_get_response_requests_exception(self, mock_get):
         self.assertIsNotNone(error)
         self.assertEqual(exception_message, error)
 
-    @mock.patch.object(ConfiguredSession, "request")
+    @mock.patch("requests.request")
     def test_get_response_generic_exception(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 500

From 62890c3ec4b775b0f92226a9c8224b130a4e2633 Mon Sep 17 00:00:00 2001
From: Justin Clift <justin@postgresql.org>
Date: Tue, 7 May 2024 03:20:05 +1000
Subject: [PATCH 02/12] Revert "Remove deprecated advocate package (#6944)"

This reverts commit bd115e7f5fd5127c081f17fcfec2cad797407454, as
it turns out to be a useful security feature.

In order to remove this in a better way, we'll need to replace it
with something that provides equivalent functionality.
---
 poetry.lock                      | 77 +++++++++++++++++++++++++++++++-
 pyproject.toml                   |  1 +
 redash/query_runner/__init__.py  | 15 +++++--
 redash/query_runner/csv.py       | 10 ++++-
 redash/query_runner/excel.py     | 10 ++++-
 redash/settings/__init__.py      |  3 ++
 redash/utils/requests_session.py | 18 ++++++++
 tests/query_runner/test_http.py  | 21 +++++----
 8 files changed, 137 insertions(+), 18 deletions(-)
 create mode 100644 redash/utils/requests_session.py

diff --git a/poetry.lock b/poetry.lock
index 6bec3f2ed1..a3ad412c46 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -17,6 +17,26 @@ PyJWT = ">=1.0.0,<3"
 python-dateutil = ">=2.1.0,<3"
 requests = ">=2.0.0,<3"
 
+[[package]]
+name = "advocate"
+version = "1.0.0"
+description = "A wrapper around the requests library for safely making HTTP requests on behalf of a third party"
+optional = false
+python-versions = "*"
+files = [
+    {file = "advocate-1.0.0-py2.py3-none-any.whl", hash = "sha256:e8b340e49fadc0e416fbc9e81ef52d74858ccad16357dabde6cf9d99a7407d70"},
+    {file = "advocate-1.0.0.tar.gz", hash = "sha256:1bf1170e41334279996580329c594e017540ab0eaf7a152323e743f0a85a353d"},
+]
+
+[package.dependencies]
+ndg-httpsclient = "*"
+netifaces = ">=0.10.5"
+pyasn1 = "*"
+pyopenssl = "*"
+requests = ">=2.18.0,<3.0"
+six = "*"
+urllib3 = ">=1.22,<2.0"
+
 [[package]]
 name = "alembic"
 version = "1.13.1"
@@ -2388,6 +2408,61 @@ files = [
     {file = "mysqlclient-2.1.1.tar.gz", hash = "sha256:828757e419fb11dd6c5ed2576ec92c3efaa93a0f7c39e263586d1ee779c3d782"},
 ]
 
+[[package]]
+name = "ndg-httpsclient"
+version = "0.5.1"
+description = "Provides enhanced HTTPS support for httplib and urllib2 using PyOpenSSL"
+optional = false
+python-versions = ">=2.7,<3.0.dev0 || >=3.4.dev0"
+files = [
+    {file = "ndg_httpsclient-0.5.1-py2-none-any.whl", hash = "sha256:d2c7225f6a1c6cf698af4ebc962da70178a99bcde24ee6d1961c4f3338130d57"},
+    {file = "ndg_httpsclient-0.5.1-py3-none-any.whl", hash = "sha256:dd174c11d971b6244a891f7be2b32ca9853d3797a72edb34fa5d7b07d8fff7d4"},
+    {file = "ndg_httpsclient-0.5.1.tar.gz", hash = "sha256:d72faed0376ab039736c2ba12e30695e2788c4aa569c9c3e3d72131de2592210"},
+]
+
+[package.dependencies]
+pyasn1 = ">=0.1.1"
+PyOpenSSL = "*"
+
+[[package]]
+name = "netifaces"
+version = "0.11.0"
+description = "Portable network interface information."
+optional = false
+python-versions = "*"
+files = [
+    {file = "netifaces-0.11.0-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eb4813b77d5df99903af4757ce980a98c4d702bbcb81f32a0b305a1537bdf0b1"},
+    {file = "netifaces-0.11.0-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:5f9ca13babe4d845e400921973f6165a4c2f9f3379c7abfc7478160e25d196a4"},
+    {file = "netifaces-0.11.0-cp27-cp27m-win32.whl", hash = "sha256:7dbb71ea26d304e78ccccf6faccef71bb27ea35e259fb883cfd7fd7b4f17ecb1"},
+    {file = "netifaces-0.11.0-cp27-cp27m-win_amd64.whl", hash = "sha256:0f6133ac02521270d9f7c490f0c8c60638ff4aec8338efeff10a1b51506abe85"},
+    {file = "netifaces-0.11.0-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:08e3f102a59f9eaef70948340aeb6c89bd09734e0dca0f3b82720305729f63ea"},
+    {file = "netifaces-0.11.0-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c03fb2d4ef4e393f2e6ffc6376410a22a3544f164b336b3a355226653e5efd89"},
+    {file = "netifaces-0.11.0-cp34-cp34m-win32.whl", hash = "sha256:73ff21559675150d31deea8f1f8d7e9a9a7e4688732a94d71327082f517fc6b4"},
+    {file = "netifaces-0.11.0-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815eafdf8b8f2e61370afc6add6194bd5a7252ae44c667e96c4c1ecf418811e4"},
+    {file = "netifaces-0.11.0-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:50721858c935a76b83dd0dd1ab472cad0a3ef540a1408057624604002fcfb45b"},
+    {file = "netifaces-0.11.0-cp35-cp35m-win32.whl", hash = "sha256:c9a3a47cd3aaeb71e93e681d9816c56406ed755b9442e981b07e3618fb71d2ac"},
+    {file = "netifaces-0.11.0-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:aab1dbfdc55086c789f0eb37affccf47b895b98d490738b81f3b2360100426be"},
+    {file = "netifaces-0.11.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c37a1ca83825bc6f54dddf5277e9c65dec2f1b4d0ba44b8fd42bc30c91aa6ea1"},
+    {file = "netifaces-0.11.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:28f4bf3a1361ab3ed93c5ef360c8b7d4a4ae060176a3529e72e5e4ffc4afd8b0"},
+    {file = "netifaces-0.11.0-cp36-cp36m-win32.whl", hash = "sha256:2650beee182fed66617e18474b943e72e52f10a24dc8cac1db36c41ee9c041b7"},
+    {file = "netifaces-0.11.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cb925e1ca024d6f9b4f9b01d83215fd00fe69d095d0255ff3f64bffda74025c8"},
+    {file = "netifaces-0.11.0-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:84e4d2e6973eccc52778735befc01638498781ce0e39aa2044ccfd2385c03246"},
+    {file = "netifaces-0.11.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:18917fbbdcb2d4f897153c5ddbb56b31fa6dd7c3fa9608b7e3c3a663df8206b5"},
+    {file = "netifaces-0.11.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:48324183af7f1bc44f5f197f3dad54a809ad1ef0c78baee2c88f16a5de02c4c9"},
+    {file = "netifaces-0.11.0-cp37-cp37m-win32.whl", hash = "sha256:8f7da24eab0d4184715d96208b38d373fd15c37b0dafb74756c638bd619ba150"},
+    {file = "netifaces-0.11.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2479bb4bb50968089a7c045f24d120f37026d7e802ec134c4490eae994c729b5"},
+    {file = "netifaces-0.11.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:3ecb3f37c31d5d51d2a4d935cfa81c9bc956687c6f5237021b36d6fdc2815b2c"},
+    {file = "netifaces-0.11.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:96c0fe9696398253f93482c84814f0e7290eee0bfec11563bd07d80d701280c3"},
+    {file = "netifaces-0.11.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c92ff9ac7c2282009fe0dcb67ee3cd17978cffbe0c8f4b471c00fe4325c9b4d4"},
+    {file = "netifaces-0.11.0-cp38-cp38-win32.whl", hash = "sha256:d07b01c51b0b6ceb0f09fc48ec58debd99d2c8430b09e56651addeaf5de48048"},
+    {file = "netifaces-0.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:469fc61034f3daf095e02f9f1bbac07927b826c76b745207287bc594884cfd05"},
+    {file = "netifaces-0.11.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5be83986100ed1fdfa78f11ccff9e4757297735ac17391b95e17e74335c2047d"},
+    {file = "netifaces-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:54ff6624eb95b8a07e79aa8817288659af174e954cca24cdb0daeeddfc03c4ff"},
+    {file = "netifaces-0.11.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:841aa21110a20dc1621e3dd9f922c64ca64dd1eb213c47267a2c324d823f6c8f"},
+    {file = "netifaces-0.11.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e76c7f351e0444721e85f975ae92718e21c1f361bda946d60a214061de1f00a1"},
+    {file = "netifaces-0.11.0.tar.gz", hash = "sha256:043a79146eb2907edf439899f262b3dfe41717d34124298ed281139a8b93ca32"},
+]
+
 [[package]]
 name = "nodeenv"
 version = "1.8.0"
@@ -5225,4 +5300,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.11"
-content-hash = "10482b8930a9e3c91c95dc79105f8bc35d9f4440e112f41fba61b74ed7ae0871"
+content-hash = "e7985ee5c3ca3a4389b4e85fda033a9b3b867dbbe4b4a7fca8ea5c35fc401148"
diff --git a/pyproject.toml b/pyproject.toml
index 8684dc3e81..e5cc871457 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,6 +23,7 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = ">=3.8,<3.11"
+advocate = "1.0.0"
 aniso8601 = "8.0.0"
 authlib = "0.15.5"
 backoff = "2.2.1"
diff --git a/redash/query_runner/__init__.py b/redash/query_runner/__init__.py
index 14a42cc3ca..10d6d6edf5 100644
--- a/redash/query_runner/__init__.py
+++ b/redash/query_runner/__init__.py
@@ -3,13 +3,17 @@
 from contextlib import ExitStack
 from functools import wraps
 
-import requests
 import sqlparse
 from dateutil import parser
 from rq.timeouts import JobTimeoutException
 from sshtunnel import open_tunnel
 
 from redash import settings, utils
+from redash.utils.requests_session import (
+    UnacceptableAddressException,
+    requests_or_advocate,
+    requests_session,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -375,7 +379,7 @@ def get_response(self, url, auth=None, http_method="get", **kwargs):
         error = None
         response = None
         try:
-            response = requests.request(http_method, url, auth=auth, **kwargs)
+            response = requests_session.request(http_method, url, auth=auth, **kwargs)
             # Raise a requests HTTP exception with the appropriate reason
             # for 4xx and 5xx response status codes which is later caught
             # and passed back.
@@ -385,11 +389,14 @@ def get_response(self, url, auth=None, http_method="get", **kwargs):
             if response.status_code != 200:
                 error = "{} ({}).".format(self.response_error, response.status_code)
 
-        except requests.HTTPError as exc:
+        except requests_or_advocate.HTTPError as exc:
             logger.exception(exc)
             error = "Failed to execute query. "
             f"Return Code: {response.status_code} Reason: {response.text}"
-        except requests.RequestException as exc:
+        except UnacceptableAddressException as exc:
+            logger.exception(exc)
+            error = "Can't query private addresses."
+        except requests_or_advocate.RequestException as exc:
             # Catch all other requests exceptions and return the error.
             logger.exception(exc)
             error = str(exc)
diff --git a/redash/query_runner/csv.py b/redash/query_runner/csv.py
index 4c72f3c3fa..3d3cf61b9c 100644
--- a/redash/query_runner/csv.py
+++ b/redash/query_runner/csv.py
@@ -1,10 +1,13 @@
 import io
 import logging
 
-import requests
 import yaml
 
 from redash.query_runner import BaseQueryRunner, NotSupported, register
+from redash.utils.requests_session import (
+    UnacceptableAddressException,
+    requests_or_advocate,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -56,7 +59,7 @@ def run_query(self, query, user):
             pass
 
         try:
-            response = requests.get(url=path, headers={"User-agent": ua})
+            response = requests_or_advocate.get(url=path, headers={"User-agent": ua})
             workbook = pd.read_csv(io.BytesIO(response.content), sep=",", **args)
 
             df = workbook.copy()
@@ -96,6 +99,9 @@ def run_query(self, query, user):
         except KeyboardInterrupt:
             error = "Query cancelled by user."
             data = None
+        except UnacceptableAddressException:
+            error = "Can't query private addresses."
+            data = None
         except Exception as e:
             error = "Error reading {0}. {1}".format(path, str(e))
             data = None
diff --git a/redash/query_runner/excel.py b/redash/query_runner/excel.py
index 5606149b31..488632e022 100644
--- a/redash/query_runner/excel.py
+++ b/redash/query_runner/excel.py
@@ -1,9 +1,12 @@
 import logging
 
-import requests
 import yaml
 
 from redash.query_runner import BaseQueryRunner, NotSupported, register
+from redash.utils.requests_session import (
+    UnacceptableAddressException,
+    requests_or_advocate,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -54,7 +57,7 @@ def run_query(self, query, user):
             pass
 
         try:
-            response = requests.get(url=path, headers={"User-agent": ua})
+            response = requests_or_advocate.get(url=path, headers={"User-agent": ua})
             workbook = pd.read_excel(response.content, **args)
 
             df = workbook.copy()
@@ -94,6 +97,9 @@ def run_query(self, query, user):
         except KeyboardInterrupt:
             error = "Query cancelled by user."
             data = None
+        except UnacceptableAddressException:
+            error = "Can't query private addresses."
+            data = None
         except Exception as e:
             error = "Error reading {0}. {1}".format(path, str(e))
             data = None
diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py
index 449f773a19..75c438cff2 100644
--- a/redash/settings/__init__.py
+++ b/redash/settings/__init__.py
@@ -71,6 +71,9 @@
 # Whether file downloads are enforced or not.
 ENFORCE_FILE_SAVE = parse_boolean(os.environ.get("REDASH_ENFORCE_FILE_SAVE", "true"))
 
+# Whether api calls using the json query runner will block private addresses
+ENFORCE_PRIVATE_ADDRESS_BLOCK = parse_boolean(os.environ.get("REDASH_ENFORCE_PRIVATE_IP_BLOCK", "true"))
+
 # Whether to use secure cookies by default.
 COOKIES_SECURE = parse_boolean(os.environ.get("REDASH_COOKIES_SECURE", str(ENFORCE_HTTPS)))
 # Whether the session cookie is set to secure.
diff --git a/redash/utils/requests_session.py b/redash/utils/requests_session.py
new file mode 100644
index 0000000000..c562894181
--- /dev/null
+++ b/redash/utils/requests_session.py
@@ -0,0 +1,18 @@
+from advocate.exceptions import UnacceptableAddressException  # noqa: F401
+
+from redash import settings
+
+if settings.ENFORCE_PRIVATE_ADDRESS_BLOCK:
+    import advocate as requests_or_advocate
+else:
+    import requests as requests_or_advocate
+
+
+class ConfiguredSession(requests_or_advocate.Session):
+    def request(self, *args, **kwargs):
+        if not settings.REQUESTS_ALLOW_REDIRECTS:
+            kwargs.update({"allow_redirects": False})
+        return super().request(*args, **kwargs)
+
+
+requests_session = ConfiguredSession()
diff --git a/tests/query_runner/test_http.py b/tests/query_runner/test_http.py
index d3e439ccb9..1fa361cb92 100644
--- a/tests/query_runner/test_http.py
+++ b/tests/query_runner/test_http.py
@@ -1,9 +1,12 @@
 from unittest import TestCase
 
 import mock
-import requests
 
 from redash.query_runner import BaseHTTPQueryRunner
+from redash.utils.requests_session import (
+    ConfiguredSession,
+    requests_or_advocate,
+)
 
 
 class RequiresAuthQueryRunner(BaseHTTPQueryRunner):
@@ -34,7 +37,7 @@ def test_get_auth_empty_requires_authentication(self):
         query_runner = RequiresAuthQueryRunner({})
         self.assertRaisesRegex(ValueError, "Username and Password required", query_runner.get_auth)
 
-    @mock.patch("requests.request")
+    @mock.patch.object(ConfiguredSession, "request")
     def test_get_response_success(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 200
@@ -48,7 +51,7 @@ def test_get_response_success(self, mock_get):
         self.assertEqual(response.status_code, 200)
         self.assertIsNone(error)
 
-    @mock.patch("requests.request")
+    @mock.patch.object(ConfiguredSession, "request")
     def test_get_response_success_custom_auth(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 200
@@ -63,7 +66,7 @@ def test_get_response_success_custom_auth(self, mock_get):
         self.assertEqual(response.status_code, 200)
         self.assertIsNone(error)
 
-    @mock.patch("requests.request")
+    @mock.patch.object(ConfiguredSession, "request")
     def test_get_response_failure(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 301
@@ -76,12 +79,12 @@ def test_get_response_failure(self, mock_get):
         mock_get.assert_called_once_with("get", url, auth=None)
         self.assertIn(query_runner.response_error, error)
 
-    @mock.patch("requests.request")
+    @mock.patch.object(ConfiguredSession, "request")
     def test_get_response_httperror_exception(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 500
         mock_response.text = "Server Error"
-        http_error = requests.HTTPError()
+        http_error = requests_or_advocate.HTTPError()
         mock_response.raise_for_status.side_effect = http_error
         mock_get.return_value = mock_response
 
@@ -92,13 +95,13 @@ def test_get_response_httperror_exception(self, mock_get):
         self.assertIsNotNone(error)
         self.assertIn("Failed to execute query", error)
 
-    @mock.patch("requests.request")
+    @mock.patch.object(ConfiguredSession, "request")
     def test_get_response_requests_exception(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 500
         mock_response.text = "Server Error"
         exception_message = "Some requests exception"
-        requests_exception = requests.RequestException(exception_message)
+        requests_exception = requests_or_advocate.RequestException(exception_message)
         mock_response.raise_for_status.side_effect = requests_exception
         mock_get.return_value = mock_response
 
@@ -109,7 +112,7 @@ def test_get_response_requests_exception(self, mock_get):
         self.assertIsNotNone(error)
         self.assertEqual(exception_message, error)
 
-    @mock.patch("requests.request")
+    @mock.patch.object(ConfiguredSession, "request")
     def test_get_response_generic_exception(self, mock_get):
         mock_response = mock.Mock()
         mock_response.status_code = 500

From 45691911130a0adf29f530fa9ac5ec19b7286120 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 7 May 2024 07:14:20 +1000
Subject: [PATCH 03/12] Bump jinja2 from 3.1.3 to 3.1.4 (#6951)

Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.3 to 3.1.4.
- [Release notes](https://github.com/pallets/jinja/releases)
- [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/jinja/compare/3.1.3...3.1.4)

---
updated-dependencies:
- dependency-name: jinja2
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 poetry.lock    | 9 +++++----
 pyproject.toml | 2 +-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index a3ad412c46..a978f02fd7 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1984,13 +1984,13 @@ testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"]
 
 [[package]]
 name = "jinja2"
-version = "3.1.3"
+version = "3.1.4"
 description = "A very fast and expressive template engine."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"},
-    {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"},
+    {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"},
+    {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"},
 ]
 
 [package.dependencies]
@@ -3825,6 +3825,7 @@ files = [
     {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
     {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
+    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
     {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
     {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
     {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
@@ -5300,4 +5301,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.11"
-content-hash = "e7985ee5c3ca3a4389b4e85fda033a9b3b867dbbe4b4a7fca8ea5c35fc401148"
+content-hash = "a65ce99c3f677c9c669cf7bbd385cd6c5980b47dcfa427739ba0d9c4e14f8120"
diff --git a/pyproject.toml b/pyproject.toml
index e5cc871457..cc91f9309c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -46,7 +46,7 @@ greenlet = "2.0.2"
 gunicorn = "22.0.0"
 httplib2 = "0.19.0"
 itsdangerous = "2.1.2"
-jinja2 = "3.1.3"
+jinja2 = "3.1.4"
 jsonschema = "3.1.1"
 markupsafe = "2.1.1"
 maxminddb-geolite2 = "2018.703"

From 1b946b59ec6cee6a042c48fa22d87d452e988042 Mon Sep 17 00:00:00 2001
From: Taehyung Lim <ehooizlo@gmail.com>
Date: Fri, 10 May 2024 20:32:52 +0900
Subject: [PATCH 04/12] sync .nvmrc with workflow (#6958)

---
 .nvmrc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.nvmrc b/.nvmrc
index 53d0020fde..3f430af82b 100644
--- a/.nvmrc
+++ b/.nvmrc
@@ -1 +1 @@
-v16.20.1
+v18

From 753ea846ff656b0c1bc4461b61951696aaafb5f5 Mon Sep 17 00:00:00 2001
From: Eric Radman <eradman@starfishstorage.com>
Date: Tue, 14 May 2024 07:54:51 -0400
Subject: [PATCH 05/12] Revert CI workflow (1 of 2) (#6965)

---
 .github/workflows/ci.yml            |  79 ++++++-----------
 .github/workflows/preview-image.yml | 129 +++++++---------------------
 2 files changed, 58 insertions(+), 150 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5f62f01e47..e8098f5b0e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,29 +3,12 @@ on:
   push:
     branches:
       - master
-    tags:
-      - '*'
   pull_request_target:
     branches:
       - master
 env:
-  CYPRESS_COVERAGE: "true"
   NODE_VERSION: 18
   YARN_VERSION: 1.22.22
-  REDASH_COOKIE_SECRET: 2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF
-  REDASH_SECRET_KEY: 2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF
-  COMPOSE_DOCKER_CLI_BUILD: 1
-  DOCKER_BUILDKIT: 1
-  FRONTEND_BUILD_MODE: 1
-  INSTALL_GROUPS: main,all_ds,dev
-  PERCY_BRANCH: ${{github.head_ref || github.ref_name}}
-  PERCY_COMMIT: ${{github.sha}}
-  PERCY_PULL_REQUEST: ${{github.event.number}}
-  COMMIT_INFO_BRANCH: ${{github.head_ref || github.ref_name}}
-  COMMIT_INFO_MESSAGE: ${{github.event.head_commit.message}}
-  COMMIT_INFO_AUTHOR: ${{github.event.pull_request.user.login}}
-  COMMIT_INFO_SHA: ${{github.sha}}
-  COMMIT_INFO_REMOTE: ${{github.server_url}}/${{github.repository}}
 jobs:
   backend-lint:
     runs-on: ubuntu-22.04
@@ -40,7 +23,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.8'
-      - run: sudo pip install black==24.3.0 ruff==0.1.9
+      - run: sudo pip install black==23.1.0 ruff==0.0.287
       - run: ruff check .
       - run: black --check .
 
@@ -48,7 +31,10 @@ jobs:
     runs-on: ubuntu-22.04
     needs: backend-lint
     env:
-      FRONTEND_BUILD_MODE: 0
+      COMPOSE_FILE: .ci/compose.ci.yaml
+      COMPOSE_PROJECT_NAME: redash
+      COMPOSE_DOCKER_CLI_BUILD: 1
+      DOCKER_BUILDKIT: 1
     steps:
       - if: github.event.pull_request.mergeable == 'false'
         name: Exit if PR is not mergeable
@@ -60,16 +46,15 @@ jobs:
       - name: Build Docker Images
         run: |
           set -x
-          touch .env
-          docker compose build
+          docker compose build --build-arg install_groups="main,all_ds,dev" --build-arg skip_frontend_build=true
           docker compose up -d
           sleep 10
       - name: Create Test Database
-        run: docker compose run --rm postgres psql -h postgres -U postgres -c "create database tests;"
+        run: docker compose -p redash run --rm postgres psql -h postgres -U postgres -c "create database tests;"
       - name: List Enabled Query Runners
-        run: docker compose run --rm server manage ds list_types
+        run: docker compose -p redash run --rm redash manage ds list_types
       - name: Run Tests
-        run: docker compose run --name tests server tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/
+        run: docker compose -p redash run --name tests redash tests --junitxml=junit.xml --cov-report=xml --cov=redash --cov-config=.coveragerc tests/
       - name: Copy Test Results
         run: |
           mkdir -p /tmp/test-results/unit-tests
@@ -77,17 +62,15 @@ jobs:
           docker cp tests:/app/junit.xml /tmp/test-results/unit-tests/results.xml
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
       - name: Store Test Results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: backend-test-results
+          name: test-results
           path: /tmp/test-results
       - name: Store Coverage Results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: backend-coverage
+          name: coverage
           path: coverage.xml
 
   frontend-lint:
@@ -107,14 +90,13 @@ jobs:
       - name: Install Dependencies
         run: |
           npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean
-          yarn --frozen-lockfile --network-concurrency 1
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
       - name: Run Lint
         run: yarn lint:ci
       - name: Store Test Results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: frontend-test-results
+          name: test-results
           path: /tmp/test-results
 
   frontend-unit-tests:
@@ -135,24 +117,21 @@ jobs:
       - name: Install Dependencies
         run: |
           npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean
-          yarn --frozen-lockfile --network-concurrency 1
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
       - name: Run App Tests
         run: yarn test
       - name: Run Visualizations Tests
-        run: |
-          cd viz-lib
-          yarn test
+        run: cd viz-lib && yarn test
       - run: yarn lint
 
   frontend-e2e-tests:
     runs-on: ubuntu-22.04
     needs: frontend-lint
     env:
+      COMPOSE_FILE: .ci/compose.cypress.yaml
+      COMPOSE_PROJECT_NAME: cypress
       CYPRESS_INSTALL_BINARY: 0
       PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: 1
-      INSTALL_GROUPS: main
-      COMPOSE_PROFILES: e2e
       PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
       CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
       CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
@@ -168,16 +147,17 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
           cache: 'yarn'
+      - name: Enable Code Coverage Report For Master Branch
+        if: endsWith(github.ref, '/master')
+        run: |
+          echo "CODE_COVERAGE=true" >> "$GITHUB_ENV"
       - name: Install Dependencies
         run: |
           npm install --global --force yarn@$YARN_VERSION
-          yarn cache clean
-          yarn --frozen-lockfile --network-concurrency 1
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
       - name: Setup Redash Server
         run: |
           set -x
-          touch .env
-          yarn build
           yarn cypress build
           yarn cypress start -- --skip-db-seed
           docker compose run cypress yarn cypress db-seed
@@ -189,12 +169,7 @@ jobs:
       - name: Copy Code Coverage Results
         run: docker cp cypress:/usr/src/app/coverage ./coverage || true
       - name: Store Coverage Results
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: frontend-coverage
+          name: coverage
           path: coverage
-      - uses: actions/upload-artifact@v4
-        with:
-          name: frontend
-          path: client/dist
-          retention-days: 1
diff --git a/.github/workflows/preview-image.yml b/.github/workflows/preview-image.yml
index 9a691fb6f4..ee81a6911f 100644
--- a/.github/workflows/preview-image.yml
+++ b/.github/workflows/preview-image.yml
@@ -1,20 +1,15 @@
 name: Preview Image
 on:
-  workflow_run:
-    workflows: 
-      - Tests
-    types:
-      - completed
-    branches:
-      - master
+  push:
+    tags:
+      - '*-dev'
 
 env:
-  DOCKER_REPO: redash
+  NODE_VERSION: 18
 
 jobs:
   build-skip-check:
     runs-on: ubuntu-22.04
-    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     outputs:
       skip: ${{ steps.skip-check.outputs.skip }}
     steps:
@@ -37,118 +32,56 @@ jobs:
     runs-on: ubuntu-22.04
     needs:
       - build-skip-check
-    outputs:
-      version: ${{ steps.version.outputs.VERSION_TAG }}
-      repo: ${{ steps.version.outputs.DOCKER_REPO }}
     if: needs.build-skip-check.outputs.skip == 'false'
-    strategy:
-      fail-fast: false
-      matrix:
-        platform:
-          - linux/amd64
-          - linux/arm64
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
           ref: ${{ github.event.push.after }}
-      - uses: dawidd6/action-download-artifact@v3
+
+      - uses: actions/setup-node@v4
         with:
-          name: frontend
-          workflow: ci.yml
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          run_id: ${{ github.event.workflow_run.id }}
-          path: client/dist
+          node-version: ${{ env.NODE_VERSION }}
+          cache: 'yarn'
+
+      - name: Install Dependencies
+        run: |
+          npm install --global --force yarn@1.22.22
+          yarn cache clean && yarn --frozen-lockfile --network-concurrency 1
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:
           username: ${{ vars.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
+
       - name: Set version
         id: version
         run: |
           set -x
-          VERSION=$(jq -r .version package.json)
-          FULL_VERSION=${VERSION}-b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}
-          sed -ri "s/^__version__ = ([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py
-          sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json
-          echo "VERSION_TAG=$FULL_VERSION" >> "$GITHUB_OUTPUT"
-          platform=${{ matrix.platform }}
-          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-          echo "SCOPE=${platform//\//-}" >> $GITHUB_ENV
-          if [[ "${{ vars.DOCKER_REPO }}" != "" ]]; then
-            echo "DOCKER_REPO=${{ vars.DOCKER_REPO }}" >> $GITHUB_ENV
-            echo "DOCKER_REPO=${{ vars.DOCKER_REPO }}" >> $GITHUB_OUTPUT
-          else
-            echo "DOCKER_REPO=${DOCKER_REPO}" >> $GITHUB_ENV
-            echo "DOCKER_REPO=${DOCKER_REPO}" >> $GITHUB_OUTPUT
-          fi
+          .ci/update_version
+          VERSION_TAG=$(jq -r .version package.json)
+          echo "VERSION_TAG=$VERSION_TAG" >> "$GITHUB_OUTPUT"
+
       - name: Build and push preview image to Docker Hub
-        uses: docker/build-push-action@v5
-        id: build
+        uses: docker/build-push-action@v4
         with:
           push: true
+          tags: |
+            redash/redash:preview
+            redash/preview:${{ steps.version.outputs.VERSION_TAG }}
           context: .
-          cache-from: type=gha,scope=${{ env.SCOPE }}
-          cache-to: type=gha,mode=max,scope=${{ env.SCOPE }}
-          platforms: ${{ matrix.platform }}
-          outputs: type=image,name=${{ env.DOCKER_REPO }}/redash,push-by-digest=true,name-canonical=true,push=true
           build-args: |
-            FRONTEND_BUILD_MODE=1
+            test_all_deps=true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          platforms: linux/amd64
         env:
           DOCKER_CONTENT_TRUST: true
-      - name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      - name: Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ env.PLATFORM_PAIR }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
 
-  publish-docker-manifest:
-    runs-on: ubuntu-22.04
-    needs:
-      - build-skip-check
-      - build-docker-image
-    if: needs.build-skip-check.outputs.skip == 'false'
-    steps:
-      - name: Download digests
-        uses: actions/download-artifact@v4
-        with:
-          pattern: digests-*
-          path: /tmp/digests
-          merge-multiple: true
-      - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ needs.build-docker-image.outputs.repo }}/redash
-          tags: preview
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ vars.DOCKER_USER }}
-          password: ${{ secrets.DOCKER_PASS }}
-      - name: Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-          $(printf '${{ needs.build-docker-image.outputs.repo }}/redash@sha256:%s ' *)
-      - name: Inspect image
-        run: |
-          REDASH_IMAGE="${{ needs.build-docker-image.outputs.repo }}/redash:${{ steps.meta.outputs.version }}"
-          docker buildx imagetools inspect $REDASH_IMAGE
-      - name: Push image ${{ needs.build-docker-image.outputs.repo }}/preview image
-        run: |
-          REDASH_IMAGE="${{ needs.build-docker-image.outputs.repo }}/redash:preview"
-          PREVIEW_IMAGE="${{ needs.build-docker-image.outputs.repo }}/preview:${{ needs.build-docker-image.outputs.version }}"
-          docker buildx imagetools create --tag $PREVIEW_IMAGE $REDASH_IMAGE
+      - name: "Failure: output container logs to console"
+        if: failure()
+        run: docker compose logs

From 58cc49bc886336e28f2c85a211554440a879cd7b Mon Sep 17 00:00:00 2001
From: Eric Radman <eradman@starfishstorage.com>
Date: Tue, 14 May 2024 08:30:48 -0400
Subject: [PATCH 06/12] Revert build (2 of 2) (#6967)

---
 Dockerfile.cypress => .ci/Dockerfile.cypress  |   0
 .ci/compose.ci.yaml                           |  25 ++++
 .ci/compose.cypress.yaml                      |  73 ++++++++++
 .ci/docker_build                              |  39 ++++++
 .ci/pack                                      |   9 ++
 .ci/update_version                            |   6 +
 Dockerfile                                    |  50 +++----
 Makefile                                      |  24 +---
 bin/docker-entrypoint                         | 127 ++++++------------
 client/cypress/cypress.js                     |  10 +-
 .../integration/dashboard/sharing_spec.js     |   6 +-
 .../integration/dashboard/widget_spec.js      |   3 +-
 compose.yaml                                  |  83 +++++-------
 package.json                                  |   2 +-
 tests/handlers/test_query_results.py          |   1 -
 15 files changed, 263 insertions(+), 195 deletions(-)
 rename Dockerfile.cypress => .ci/Dockerfile.cypress (100%)
 create mode 100644 .ci/compose.ci.yaml
 create mode 100644 .ci/compose.cypress.yaml
 create mode 100755 .ci/docker_build
 create mode 100755 .ci/pack
 create mode 100755 .ci/update_version

diff --git a/Dockerfile.cypress b/.ci/Dockerfile.cypress
similarity index 100%
rename from Dockerfile.cypress
rename to .ci/Dockerfile.cypress
diff --git a/.ci/compose.ci.yaml b/.ci/compose.ci.yaml
new file mode 100644
index 0000000000..7c056d0f26
--- /dev/null
+++ b/.ci/compose.ci.yaml
@@ -0,0 +1,25 @@
+services:
+  redash:
+    build: ../
+    command: manage version
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    environment:
+      PYTHONUNBUFFERED: 0
+      REDASH_LOG_LEVEL: "INFO"
+      REDASH_REDIS_URL: "redis://redis:6379/0"
+      POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb"
+      REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres"
+      REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF"
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+  postgres:
+    image: pgautoupgrade/pgautoupgrade:latest
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"
+    restart: unless-stopped
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"
diff --git a/.ci/compose.cypress.yaml b/.ci/compose.cypress.yaml
new file mode 100644
index 0000000000..7f769ab3ef
--- /dev/null
+++ b/.ci/compose.cypress.yaml
@@ -0,0 +1,73 @@
+x-redash-service: &redash-service
+  build:
+    context: ../
+    args:
+      install_groups: "main"
+      code_coverage: ${CODE_COVERAGE}
+x-redash-environment: &redash-environment
+  REDASH_LOG_LEVEL: "INFO"
+  REDASH_REDIS_URL: "redis://redis:6379/0"
+  POSTGRES_PASSWORD: "FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb"
+  REDASH_DATABASE_URL: "postgresql://postgres:FmTKs5vX52ufKR1rd8tn4MoSP7zvCJwb@postgres/postgres"
+  REDASH_RATELIMIT_ENABLED: "false"
+  REDASH_ENFORCE_CSRF: "true"
+  REDASH_COOKIE_SECRET: "2H9gNG9obnAQ9qnR9BDTQUph6CbXKCzF"
+services:
+  server:
+    <<: *redash-service
+    command: server
+    depends_on:
+      - postgres
+      - redis
+    ports:
+      - "5000:5000"
+    environment:
+      <<: *redash-environment
+      PYTHONUNBUFFERED: 0
+  scheduler:
+    <<: *redash-service
+    command: scheduler
+    depends_on:
+      - server
+    environment:
+      <<: *redash-environment
+  worker:
+    <<: *redash-service
+    command: worker
+    depends_on:
+      - server
+    environment:
+      <<: *redash-environment
+      PYTHONUNBUFFERED: 0
+  cypress:
+    ipc: host
+    build:
+      context: ../
+      dockerfile: .ci/Dockerfile.cypress
+    depends_on:
+      - server
+      - worker
+      - scheduler
+    environment:
+      CYPRESS_baseUrl: "http://server:5000"
+      CYPRESS_coverage: ${CODE_COVERAGE}
+      PERCY_TOKEN: ${PERCY_TOKEN}
+      PERCY_BRANCH: ${CIRCLE_BRANCH}
+      PERCY_COMMIT: ${CIRCLE_SHA1}
+      PERCY_PULL_REQUEST: ${CIRCLE_PR_NUMBER}
+      COMMIT_INFO_BRANCH: ${CIRCLE_BRANCH}
+      COMMIT_INFO_MESSAGE: ${COMMIT_INFO_MESSAGE}
+      COMMIT_INFO_AUTHOR: ${CIRCLE_USERNAME}
+      COMMIT_INFO_SHA: ${CIRCLE_SHA1}
+      COMMIT_INFO_REMOTE: ${CIRCLE_REPOSITORY_URL}
+      CYPRESS_PROJECT_ID: ${CYPRESS_PROJECT_ID}
+      CYPRESS_RECORD_KEY: ${CYPRESS_RECORD_KEY}
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+  postgres:
+    image: pgautoupgrade/pgautoupgrade:latest
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"
+    restart: unless-stopped
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: "trust"
diff --git a/.ci/docker_build b/.ci/docker_build
new file mode 100755
index 0000000000..324c7e996e
--- /dev/null
+++ b/.ci/docker_build
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# This script only needs to run on the main Redash repo
+
+if [ "${GITHUB_REPOSITORY}" != "getredash/redash" ]; then
+	echo "Skipping image build for Docker Hub, as this isn't the main Redash repository"
+	exit 0
+fi
+
+if [ "${GITHUB_REF_NAME}" != "master" ] && [ "${GITHUB_REF_NAME}" != "preview-image" ]; then
+	echo "Skipping image build for Docker Hub, as this isn't the 'master' nor 'preview-image' branch"
+	exit 0
+fi
+
+if [ "x${DOCKER_USER}" = "x" ] || [ "x${DOCKER_PASS}" = "x" ]; then
+	echo "Skipping image build for Docker Hub, as the login details aren't available"
+	exit 0
+fi
+
+set -e
+VERSION=$(jq -r .version package.json)
+VERSION_TAG="$VERSION.b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}"
+
+export DOCKER_BUILDKIT=1
+export COMPOSE_DOCKER_CLI_BUILD=1
+
+docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}"
+
+DOCKERHUB_REPO="redash/redash"
+DOCKER_TAGS="-t redash/redash:preview -t redash/preview:${VERSION_TAG}"
+
+# Build the docker container
+docker build --build-arg install_groups="main,all_ds,dev" ${DOCKER_TAGS} .
+
+# Push the container to the preview build locations
+docker push "${DOCKERHUB_REPO}:preview"
+docker push "redash/preview:${VERSION_TAG}"
+
+echo "Built: ${VERSION_TAG}"
diff --git a/.ci/pack b/.ci/pack
new file mode 100755
index 0000000000..16223c5a9b
--- /dev/null
+++ b/.ci/pack
@@ -0,0 +1,9 @@
+#!/bin/bash
+NAME=redash
+VERSION=$(jq -r .version package.json)
+FULL_VERSION=$VERSION+b$CIRCLE_BUILD_NUM
+FILENAME=$NAME.$FULL_VERSION.tar.gz
+
+mkdir -p /tmp/artifacts/
+
+tar -zcv -f /tmp/artifacts/$FILENAME --exclude=".git" --exclude="optipng*" --exclude="cypress" --exclude="*.pyc" --exclude="*.pyo" --exclude="venv" *
diff --git a/.ci/update_version b/.ci/update_version
new file mode 100755
index 0000000000..53b537208c
--- /dev/null
+++ b/.ci/update_version
@@ -0,0 +1,6 @@
+#!/bin/bash
+VERSION=$(jq -r .version package.json)
+FULL_VERSION=${VERSION}+b${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}
+
+sed -ri "s/^__version__ = '([A-Za-z0-9.-]*)'/__version__ = '${FULL_VERSION}'/" redash/__init__.py
+sed -i "s/dev/${GITHUB_SHA}/" client/app/version.json
diff --git a/Dockerfile b/Dockerfile
index c4497d1335..20fdcde61d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,35 +1,30 @@
-# Controls whether to build the frontend assets
-ARG FRONTEND_BUILD_MODE=0
+FROM node:18-bookworm as frontend-builder
 
-# MODE 0: create empty files. useful for backend tests
-FROM alpine:3.19 as frontend-builder-0
-RUN \
-  mkdir -p /frontend/client/dist && \
-  touch /frontend/client/dist/multi_org.html && \
-  touch /frontend/client/dist/index.html
+RUN npm install --global --force yarn@1.22.22
 
-# MODE 1: copy static frontend from host, useful for CI to ignore building static content multiple times
-FROM alpine:3.19 as frontend-builder-1
-COPY client/dist /frontend/client/dist
+# Controls whether to build the frontend assets
+ARG skip_frontend_build
 
-# MODE 2: build static content in docker, can be used for a local development
-FROM node:18-bookworm as frontend-builder-2
-RUN npm install --global --force yarn@1.22.22
 ENV CYPRESS_INSTALL_BINARY=0
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
+
 RUN useradd -m -d /frontend redash
 USER redash
+
 WORKDIR /frontend
 COPY --chown=redash package.json yarn.lock .yarnrc /frontend/
 COPY --chown=redash viz-lib /frontend/viz-lib
 COPY --chown=redash scripts /frontend/scripts
 
-RUN yarn --frozen-lockfile --network-concurrency 1;
+# Controls whether to instrument code for coverage information
+ARG code_coverage
+ENV BABEL_ENV=${code_coverage:+test}
+
+RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn --frozen-lockfile --network-concurrency 1; fi
+
 COPY --chown=redash client /frontend/client
 COPY --chown=redash webpack.config.js /frontend/
-RUN yarn build
-
-FROM frontend-builder-${FRONTEND_BUILD_MODE} as frontend-builder
+RUN if [ "x$skip_frontend_build" = "x" ] ; then yarn build; else mkdir -p /frontend/client/dist && touch /frontend/client/dist/multi_org.html && touch /frontend/client/dist/index.html; fi
 
 FROM python:3.8-slim-bookworm
 
@@ -66,18 +61,17 @@ RUN apt-get update && \
   apt-get clean && \
   rm -rf /var/lib/apt/lists/*
 
-RUN \
-  curl https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
-  curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg && \
-  apt update && \
-  ACCEPT_EULA=Y apt install  -y --no-install-recommends msodbcsql18 && \
-  apt clean && \
-  rm -rf /var/lib/apt/lists/*
 
 ARG TARGETPLATFORM
 ARG databricks_odbc_driver_url=https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/odbc/2.6.26/SimbaSparkODBC-2.6.26.1045-Debian-64bit.zip
 RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
-  curl "$databricks_odbc_driver_url" --location --output /tmp/simba_odbc.zip \
+  curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \
+  && curl https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list \
+  && apt-get update \
+  && ACCEPT_EULA=Y apt-get install  -y --no-install-recommends msodbcsql17 \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/* \
+  && curl "$databricks_odbc_driver_url" --location --output /tmp/simba_odbc.zip \
   && chmod 600 /tmp/simba_odbc.zip \
   && unzip /tmp/simba_odbc.zip -d /tmp/simba \
   && dpkg -i /tmp/simba/*.deb \
@@ -97,8 +91,8 @@ COPY pyproject.toml poetry.lock ./
 ARG POETRY_OPTIONS="--no-root --no-interaction --no-ansi"
 # for LDAP authentication, install with `ldap3` group
 # disabled by default due to GPL license conflict
-ARG INSTALL_GROUPS="main,all_ds,dev"
-RUN /etc/poetry/bin/poetry install --only $INSTALL_GROUPS $POETRY_OPTIONS
+ARG install_groups="main,all_ds,dev"
+RUN /etc/poetry/bin/poetry install --only $install_groups $POETRY_OPTIONS
 
 COPY --chown=redash . /app
 COPY --from=frontend-builder --chown=redash /frontend/client/dist /app/client/dist
diff --git a/Makefile b/Makefile
index c6b0363543..404bfc10e1 100644
--- a/Makefile
+++ b/Makefile
@@ -1,18 +1,10 @@
-.PHONY: compose_build up test_db create_database create_db clean clean-all down tests lint backend-unit-tests frontend-unit-tests pydeps test build watch start redis-cli bash
-
-export COMPOSE_DOCKER_CLI_BUILD=1
-export DOCKER_BUILDKIT=1
-export COMPOSE_PROFILES=local
+.PHONY: compose_build up test_db create_database clean clean-all down tests lint backend-unit-tests frontend-unit-tests test build watch start redis-cli bash
 
 compose_build: .env
-	docker compose build
+	COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose build
 
 up:
-	docker compose up -d redis postgres
-	docker compose exec -u postgres postgres psql postgres --csv \
-		-1tqc "SELECT table_name FROM information_schema.tables WHERE table_name = 'organizations'" 2> /dev/null \
-		| grep -q "organizations" || make create_database
-	docker compose up -d --build
+	COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose up -d --build
 
 test_db:
 	@for i in `seq 1 5`; do \
@@ -21,11 +13,9 @@ test_db:
 	done
 	docker compose exec postgres sh -c 'psql -U postgres -c "drop database if exists tests;" && psql -U postgres -c "create database tests;"'
 
-create_db: .env
+create_database: .env
 	docker compose run server create_db
 
-create_database: create_db
-
 clean:
 	docker compose down
 	docker compose --project-name cypress down
@@ -54,12 +44,6 @@ env: .env
 format:
 	pre-commit run --all-files
 
-pydeps:
-	pip3 install wheel
-	pip3 install --upgrade black ruff launchpadlib pip setuptools
-	pip3 install poetry
-	poetry install --only main,all_ds,dev
-
 tests:
 	docker compose run server tests
 
diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint
index 48a8621a92..5e777c10ed 100755
--- a/bin/docker-entrypoint
+++ b/bin/docker-entrypoint
@@ -1,48 +1,25 @@
 #!/bin/bash
 set -e
 
-if [ -z $REDASH_REDIS_URL ]; then
-  export REDASH_REDIS_URL=redis://:${REDASH_REDIS_PASSWORD}@${REDASH_REDIS_HOSTNAME}:${REDASH_REDIS_PORT}/${REDASH_REDIS_NAME}
-fi
-
-if [ -z $REDASH_DATABASE_URL ]; then
-  export REDASH_DATABASE_URL=postgresql://${REDASH_DATABASE_USER}:${REDASH_DATABASE_PASSWORD}@${REDASH_DATABASE_HOSTNAME}:${REDASH_DATABASE_PORT}/${REDASH_DATABASE_NAME}
-fi
-
 scheduler() {
   echo "Starting RQ scheduler..."
 
-  case $REDASH_PRODUCTION in
-    true)
-      echo "Starting RQ scheduler in production mode"
-      exec ./manage.py rq scheduler
-      ;;
-    *)
-      echo "Starting RQ scheduler in dev mode"
-      exec watchmedo auto-restart \
-        --directory=./redash/ \
-        --pattern=*.py \
-        --recursive -- ./manage.py rq scheduler $QUEUES
-      ;;
-  esac
+  exec /app/manage.py rq scheduler
+}
+
+dev_scheduler() {
+  echo "Starting dev RQ scheduler..."
+
+  exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq scheduler
 }
 
 worker() {
+  echo "Starting RQ worker..."
+
   export WORKERS_COUNT=${WORKERS_COUNT:-2}
   export QUEUES=${QUEUES:-}
-  case $REDASH_PRODUCTION in
-    true)
-      echo "Starting RQ worker in production mode"
-      exec supervisord -c worker.conf
-      ;;
-    *)
-      echo "Starting RQ worker in dev mode"
-      exec watchmedo auto-restart \
-        --directory=./redash/ \
-        --pattern=*.py \
-        --recursive -- ./manage.py rq worker $QUEUES
-      ;;
-  esac
+
+  exec supervisord -c worker.conf
 }
 
 workers_healthcheck() {
@@ -58,63 +35,22 @@ workers_healthcheck() {
   fi
 }
 
+dev_worker() {
+  echo "Starting dev RQ worker..."
+
+  exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- ./manage.py rq worker $QUEUES
+}
+
 server() {
   # Recycle gunicorn workers every n-th request. See http://docs.gunicorn.org/en/stable/settings.html#max-requests for more details.
-  case $REDASH_PRODUCTION in
-    true)
-      echo "Starting Redash Server in production mode"
-      MAX_REQUESTS=${MAX_REQUESTS:-1000}
-      MAX_REQUESTS_JITTER=${MAX_REQUESTS_JITTER:-100}
-      TIMEOUT=${REDASH_GUNICORN_TIMEOUT:-60}
-      exec /usr/local/bin/gunicorn \
-        -b 0.0.0.0:5000 \
-        --name redash \
-        -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app \
-        --max-requests $MAX_REQUESTS \
-        --max-requests-jitter $MAX_REQUESTS_JITTER \
-        --timeout $TIMEOUT
-      ;;
-    *)
-      echo "Starting Redash Server in a dev mode"
-      export FLASK_DEBUG=1
-      exec /app/manage.py runserver --debugger --reload -h 0.0.0.0
-      ;;
-  esac
+  MAX_REQUESTS=${MAX_REQUESTS:-1000}
+  MAX_REQUESTS_JITTER=${MAX_REQUESTS_JITTER:-100}
+  TIMEOUT=${REDASH_GUNICORN_TIMEOUT:-60}
+  exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app --max-requests $MAX_REQUESTS --max-requests-jitter $MAX_REQUESTS_JITTER --timeout $TIMEOUT
 }
 
 create_db() {
-  REDASH_DATABASE_MIGRATE_TIMEOUT=${REDASH_DATABASE_UPGRADE_TIMEOUT:-600}
-  REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS=${REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS:-5}
-  REDASH_DATABASE_MIGRATE_RETRY_WAIT=${REDASH_DATABASE_MIGRATE_RETRY_WAIT:-10}
-  ATTEMPTS=1
-  while ((ATTEMPTS <= REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS)); do
-    echo "Creating or updating Redash database, attempt ${ATTEMPTS} of ${REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS}"
-    ATTEMPTS=$((ATTEMPTS+1))
-    timeout $REDASH_DATABASE_MIGRATE_TIMEOUT /app/manage.py database create_tables
-    timeout $REDASH_DATABASE_MIGRATE_TIMEOUT /app/manage.py db upgrade
-    STATUS=$(timeout $REDASH_DATABASE_MIGRATE_TIMEOUT /app/manage.py status 2>&1)
-    RETCODE=$?
-    case "$RETCODE" in
-      0)
-        exit 0
-        ;;
-      124)
-        echo "Status command timed out after ${REDASH_DATABASE_MIGRATE_TIMEOUT} seconds."
-        ;;
-    esac
-    case "$STATUS" in
-      *sqlalchemy.exc.OperationalError*)
-        echo "Database not yet functional, waiting."
-        ;;
-      *sqlalchemy.exc.ProgrammingError*)
-        echo "Database does not appear to be installed."
-        ;;
-    esac
-    echo "Waiting ${REDASH_DATABASE_MIGRATE_RETRY_WAIT} seconds before retrying."
-    sleep ${REDASH_DATABASE_MIGRATE_RETRY_WAIT}
-  done
-  echo "Reached ${REDASH_DATABASE_MIGRATE_MAX_ATTEMPTS} attempts, giving up."
-  exit 1
+  exec /app/manage.py database create_tables
 }
 
 help() {
@@ -125,16 +61,21 @@ help() {
 
   echo "server -- start Redash server (with gunicorn)"
   echo "worker -- start a single RQ worker"
+  echo "dev_worker -- start a single RQ worker with code reloading"
   echo "scheduler -- start an rq-scheduler instance"
+  echo "dev_scheduler -- start an rq-scheduler instance with code reloading"
   echo ""
   echo "shell -- open shell"
+  echo "dev_server -- start Flask development server with debugger and auto reload"
   echo "debug -- start Flask development server with remote debugger via ptvsd"
-  echo "create_db -- create database tables and run migrations"
+  echo "create_db -- create database tables"
   echo "manage -- CLI to manage redash"
   echo "tests -- run tests"
 }
 
 tests() {
+  export REDASH_DATABASE_URL="postgresql://postgres@postgres/tests"
+
   if [ $# -eq 0 ]; then
     TEST_ARGS=tests/
   else
@@ -160,10 +101,22 @@ case "$1" in
     shift
     scheduler
     ;;
+  dev_scheduler)
+    shift
+    dev_scheduler
+    ;;
+  dev_worker)
+    shift
+    dev_worker
+    ;;
   celery_healthcheck)
     shift
     echo "DEPRECATED: Celery has been replaced with RQ and now performs healthchecks autonomously as part of the 'worker' entrypoint."
     ;;
+  dev_server)
+    export FLASK_DEBUG=1
+    exec /app/manage.py runserver --debugger --reload -h 0.0.0.0
+    ;;
   debug)
     export FLASK_DEBUG=1
     export REMOTE_DEBUG=1
diff --git a/client/cypress/cypress.js b/client/cypress/cypress.js
index ce6cb49e78..320402f18d 100644
--- a/client/cypress/cypress.js
+++ b/client/cypress/cypress.js
@@ -43,18 +43,18 @@ function seedDatabase(seedValues) {
 
 function buildServer() {
   console.log("Building the server...");
-  execSync("docker compose build", { stdio: "inherit" });
+  execSync("docker compose -p cypress build", { stdio: "inherit" });
 }
 
 function startServer() {
   console.log("Starting the server...");
-  execSync("docker compose up -d", { stdio: "inherit" });
-  execSync("docker compose run server create_db", { stdio: "inherit" });
+  execSync("docker compose -p cypress up -d", { stdio: "inherit" });
+  execSync("docker compose -p cypress run server create_db", { stdio: "inherit" });
 }
 
 function stopServer() {
   console.log("Stopping the server...");
-  execSync("docker compose down", { stdio: "inherit" });
+  execSync("docker compose -p cypress down", { stdio: "inherit" });
 }
 
 function runCypressCI() {
@@ -68,7 +68,7 @@ function runCypressCI() {
   }
 
   execSync(
-    "docker compose run --name cypress cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run $CYPRESS_OPTIONS",
+    "COMMIT_INFO_MESSAGE=$(git show -s --format=%s) docker compose run --name cypress cypress ./node_modules/.bin/percy exec -t 300 -- ./node_modules/.bin/cypress run $CYPRESS_OPTIONS",
     { stdio: "inherit" }
   );
 }
diff --git a/client/cypress/integration/dashboard/sharing_spec.js b/client/cypress/integration/dashboard/sharing_spec.js
index 97db547da9..5e73b54e51 100644
--- a/client/cypress/integration/dashboard/sharing_spec.js
+++ b/client/cypress/integration/dashboard/sharing_spec.js
@@ -53,12 +53,11 @@ describe("Dashboard Sharing", () => {
     };
 
     const dashboardUrl = this.dashboardUrl;
-    cy.createQuery({ options }).then(({ id: queryId, name: queryName }) => {
+    cy.createQuery({ options }).then(({ id: queryId }) => {
       cy.visit(dashboardUrl);
       editDashboard();
       cy.getByTestId("AddWidgetButton").click();
       cy.getByTestId("AddWidgetDialog").within(() => {
-        cy.get("input").type(queryName);
         cy.get(`.query-selector-result[data-test="QueryId${queryId}"]`).click();
       });
       cy.contains("button", "Add to Dashboard").click();
@@ -179,12 +178,11 @@ describe("Dashboard Sharing", () => {
     };
 
     const dashboardUrl = this.dashboardUrl;
-    cy.createQuery({ options }).then(({ id: queryId, name: queryName }) => {
+    cy.createQuery({ options }).then(({ id: queryId }) => {
       cy.visit(dashboardUrl);
       editDashboard();
       cy.getByTestId("AddWidgetButton").click();
       cy.getByTestId("AddWidgetDialog").within(() => {
-        cy.get("input").type(queryName);
         cy.get(`.query-selector-result[data-test="QueryId${queryId}"]`).click();
       });
       cy.contains("button", "Add to Dashboard").click();
diff --git a/client/cypress/integration/dashboard/widget_spec.js b/client/cypress/integration/dashboard/widget_spec.js
index 875a6f5b02..2c77fff832 100644
--- a/client/cypress/integration/dashboard/widget_spec.js
+++ b/client/cypress/integration/dashboard/widget_spec.js
@@ -18,12 +18,11 @@ describe("Widget", () => {
   };
 
   it("adds widget", function() {
-    cy.createQuery().then(({ id: queryId, name: queryName }) => {
+    cy.createQuery().then(({ id: queryId }) => {
       cy.visit(this.dashboardUrl);
       editDashboard();
       cy.getByTestId("AddWidgetButton").click();
       cy.getByTestId("AddWidgetDialog").within(() => {
-        cy.get("input").type(queryName);
         cy.get(`.query-selector-result[data-test="QueryId${queryId}"]`).click();
       });
       cy.contains("button", "Add to Dashboard").click();
diff --git a/compose.yaml b/compose.yaml
index b34cd60f17..3e2485aec9 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -1,38 +1,52 @@
+# This configuration file is for the **development** setup.
+# For a production example please refer to getredash/setup repository on GitHub.
+x-redash-service: &redash-service
+  build:
+    context: .
+    args:
+      skip_frontend_build: "true"  # set to empty string to build
+  volumes:
+    - .:/app
+  env_file:
+    - .env
+x-redash-environment: &redash-environment
+  REDASH_LOG_LEVEL: "INFO"
+  REDASH_REDIS_URL: "redis://redis:6379/0"
+  REDASH_DATABASE_URL: "postgresql://postgres@postgres/postgres"
+  REDASH_RATELIMIT_ENABLED: "false"
+  REDASH_MAIL_DEFAULT_SENDER: "redash@example.com"
+  REDASH_MAIL_SERVER: "email"
+  REDASH_MAIL_PORT: 1025
+  REDASH_ENFORCE_CSRF: "true"
+  REDASH_GUNICORN_TIMEOUT: 60
+  # Set secret keys in the .env file
 services:
   server:
-    extends:
-      file: compose.base.yaml
-      service: .redash
-    command: server
+    <<: *redash-service
+    command: dev_server
     depends_on:
       - postgres
       - redis
     ports:
-      - "${REDASH_PORT:-5001}:5000"
+      - "5001:5000"
       - "5678:5678"
     environment:
+      <<: *redash-environment
       PYTHONUNBUFFERED: 0
   scheduler:
-    extends:
-      file: compose.base.yaml
-      service: .redash
-    profiles:
-      - e2e
-      - local
-    command: scheduler
+    <<: *redash-service
+    command: dev_scheduler
     depends_on:
       - server
+    environment:
+      <<: *redash-environment
   worker:
-    extends:
-      file: compose.base.yaml
-      service: .redash
-    profiles:
-      - e2e
-      - local
-    command: worker
+    <<: *redash-service
+    command: dev_worker
     depends_on:
       - server
     environment:
+      <<: *redash-environment
       PYTHONUNBUFFERED: 0
   redis:
     image: redis:7-alpine
@@ -40,42 +54,17 @@ services:
   postgres:
     image: pgautoupgrade/pgautoupgrade:latest
     ports:
-      - "${POSTGRES_PORT:-15432}:5432"
+      - "15432:5432"
     # The following turns the DB into less durable, but gains significant performance improvements for the tests run (x3
     # improvement on my personal machine). We should consider moving this into a dedicated Docker Compose configuration for
     # tests.
-    command: postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF
+    command: "postgres -c fsync=off -c full_page_writes=off -c synchronous_commit=OFF"
     restart: unless-stopped
     environment:
-      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_HOST_AUTH_METHOD: "trust"
   email:
     image: maildev/maildev
     ports:
       - "1080:1080"
       - "1025:1025"
     restart: unless-stopped
-  cypress:
-    ipc: host
-    build:
-      context: .
-      dockerfile: Dockerfile.cypress
-    profiles:
-      - e2e
-    depends_on:
-      - server
-      - worker
-      - scheduler
-    environment:
-      CYPRESS_baseUrl: http://server:5000
-      PERCY_TOKEN: ${PERCY_TOKEN:-""}
-      PERCY_BRANCH: ${PERCY_BRANCH:-""}
-      PERCY_COMMIT: ${PERCY_COMMIT:-""}
-      PERCY_PULL_REQUEST: ${PERCY_PULL_REQUEST:-}
-      COMMIT_INFO_BRANCH: ${COMMIT_INFO_BRANCH:-""}
-      COMMIT_INFO_MESSAGE: ${COMMIT_INFO_MESSAGE:-""}
-      COMMIT_INFO_AUTHOR: ${COMMIT_INFO_AUTHOR:-""}
-      COMMIT_INFO_SHA: ${COMMIT_INFO_SHA:-""}
-      COMMIT_INFO_REMOTE: ${COMMIT_INFO_REMOTE:-""}
-      CYPRESS_PROJECT_ID: ${CYPRESS_PROJECT_ID:-""}
-      CYPRESS_RECORD_KEY: ${CYPRESS_RECORD_KEY:-""}
-      CYPRESS_COVERAGE: ${CYPRESS_COVERAGE:-true}
diff --git a/package.json b/package.json
index 1d03fbbf5d..032d4bf126 100644
--- a/package.json
+++ b/package.json
@@ -24,7 +24,7 @@
     "jest": "TZ=Africa/Khartoum jest",
     "test": "run-s type-check jest",
     "test:watch": "jest --watch",
-    "cypress": "COMPOSE_PROFILES=local node client/cypress/cypress.js",
+    "cypress": "node client/cypress/cypress.js",
     "preinstall": "cd viz-lib && yarn link --link-folder ../.yarn",
     "postinstall": "(cd viz-lib && yarn --frozen-lockfile && yarn build:babel) && yarn link --link-folder ./.yarn @redash/viz"
   },
diff --git a/tests/handlers/test_query_results.py b/tests/handlers/test_query_results.py
index ea4957f147..cbecafee62 100644
--- a/tests/handlers/test_query_results.py
+++ b/tests/handlers/test_query_results.py
@@ -436,7 +436,6 @@ def test_renders_excel_file_when_rows_have_missing_columns(self):
 
 class TestJobResource(BaseTestCase):
     def test_cancels_queued_queries(self):
-
         query = self.factory.create_query()
         job_id = self.make_request(
             "post",

From 408ba78bd093db9407922ff8d7491fb62ecc42bc Mon Sep 17 00:00:00 2001
From: Eric Radman <eradman@starfishstorage.com>
Date: Tue, 14 May 2024 11:55:32 -0400
Subject: [PATCH 07/12] Update MSSQL OBDC driver to v18 (#6968)

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 20fdcde61d..b7d5ed23d7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -68,7 +68,7 @@ RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
   curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft-prod.gpg \
   && curl https://packages.microsoft.com/config/debian/12/prod.list > /etc/apt/sources.list.d/mssql-release.list \
   && apt-get update \
-  && ACCEPT_EULA=Y apt-get install  -y --no-install-recommends msodbcsql17 \
+  && ACCEPT_EULA=Y apt-get install  -y --no-install-recommends msodbcsql18 \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* \
   && curl "$databricks_odbc_driver_url" --location --output /tmp/simba_odbc.zip \

From f3a323695f541dfc65db47c9752b499a38cfa966 Mon Sep 17 00:00:00 2001
From: Taehyung Lim <ehooizlo@gmail.com>
Date: Wed, 15 May 2024 01:26:28 +0900
Subject: [PATCH 08/12] Bump pyodbc from 4.0.28 to 5.1.0 (#6962)

---
 poetry.lock    | 52 +++++++++++++++++++++++++++++++++-----------------
 pyproject.toml |  3 ++-
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index a978f02fd7..42b44a606f 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -3536,24 +3536,42 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"]
 
 [[package]]
 name = "pyodbc"
-version = "4.0.28"
-description = "DB API Module for ODBC"
+version = "5.1.0"
+description = "DB API module for ODBC"
 optional = false
-python-versions = "*"
+python-versions = ">=3.8"
 files = [
-    {file = "pyodbc-4.0.28-cp27-cp27m-win32.whl", hash = "sha256:2217eb01091a207a9ffa457c49a63a1d0eb8514c810a23b901518348422fcf65"},
-    {file = "pyodbc-4.0.28-cp27-cp27m-win_amd64.whl", hash = "sha256:ae35c455bfbadc631ee20df6657bfda0779bdc80badfd9d13741433dd78785e6"},
-    {file = "pyodbc-4.0.28-cp27-none-macosx_10_15_x86_64.whl", hash = "sha256:f37f26ae909101465a085ef51b9dde35afc93b7c7e38c25b61b124b110aa9998"},
-    {file = "pyodbc-4.0.28-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:5d1abca8f5bdab1515e300d05c63c25d072a123c7089a554290b5b9e83168eb6"},
-    {file = "pyodbc-4.0.28-cp36-cp36m-win32.whl", hash = "sha256:c25e525e0576b1dfa067d3a6530e046a24006d89715026d2d5dbf6d4290093b9"},
-    {file = "pyodbc-4.0.28-cp36-cp36m-win_amd64.whl", hash = "sha256:259b2554d2b8c9a6247871fec741b526f0b63a0e42676bd8f210e214a3015129"},
-    {file = "pyodbc-4.0.28-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ad9aa2a851242109141e4275c2a9b4d4379e00288959acd877501ee90aa3955"},
-    {file = "pyodbc-4.0.28-cp37-cp37m-win32.whl", hash = "sha256:2908f73e5a374437fd7a38f14b09f2b96d742235bf2f819fb697f8922e35ddda"},
-    {file = "pyodbc-4.0.28-cp37-cp37m-win_amd64.whl", hash = "sha256:a1a1687edef4319ae533e1d789c6c8241459f04af9e4db76e6e4045c530239de"},
-    {file = "pyodbc-4.0.28-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4f3c788d231907f45ea329cd245b398b165d9d28809f55814240eea775a6b1cd"},
-    {file = "pyodbc-4.0.28-cp38-cp38-win32.whl", hash = "sha256:93e495c51a5db027c2f7ee2c2c3fe9d6ea86b3a61392c7c8961a1818951868c8"},
-    {file = "pyodbc-4.0.28-cp38-cp38-win_amd64.whl", hash = "sha256:49ba851be2d9d07cc1472b43febc93e3362c1e09ceb3eac84693a6690d090165"},
-    {file = "pyodbc-4.0.28.tar.gz", hash = "sha256:510643354c4c687ed96bf7e7cec4d02d6c626ecf3e18696f5a0228dd6d11b769"},
+    {file = "pyodbc-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:02fe9821711a2d14415eaeb4deab471d2c8b7034b107e524e414c0e133c42248"},
+    {file = "pyodbc-5.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2cbdbd019756285dc44bc35238a3ed8dfaa454e8c8b2c3462f1710cfeebfb290"},
+    {file = "pyodbc-5.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84df3bbce9bafe65abd25788d55c9f1da304f6115d70f25758ff8c85f3ce0517"},
+    {file = "pyodbc-5.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:218bb75d4bc67075529a65ce8ec7daeed1d83c33dd7410450fbf68d43d184d28"},
+    {file = "pyodbc-5.1.0-cp310-cp310-win32.whl", hash = "sha256:eae576b3b67d21d6f237e18bb5f3df8323a2258f52c3e3afeef79269704072a9"},
+    {file = "pyodbc-5.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:c3b65343557f4c7753204e06f4c82c97ed212a636501f4bc27c5ce0e549eb3e8"},
+    {file = "pyodbc-5.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aa6f46377da303bf79bcb4b559899507df4b2559f30dcfdf191358ee4b99f3ab"},
+    {file = "pyodbc-5.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b19d7f44cfee89901e482f554a88177e83fae76b03c3f830e0023a195d840220"},
+    {file = "pyodbc-5.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c36448322f8d6479d87c528cf52401a6ea4f509b9637750b67340382b4e1b40"},
+    {file = "pyodbc-5.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c5e0cb79222aad4b31a3602e39b242683c29c6221a16ed43f45f18fd0b73659"},
+    {file = "pyodbc-5.1.0-cp311-cp311-win32.whl", hash = "sha256:92caed9d445815ed3f7e5a1249e29a4600ebc1e99404df81b6ed7671074c9227"},
+    {file = "pyodbc-5.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a1bd14633e91b7a9814f4fd944c9ebb89fb7f1fd4710c4e3999b5ef041536347"},
+    {file = "pyodbc-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d3d9cc4af703c4817b6e604315910b0cf5dcb68056d52b25ca072dd59c52dcbc"},
+    {file = "pyodbc-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:406b8fa2133a7b6a713aa5187dba2d08cf763b5884606bed77610a7660fdfabe"},
+    {file = "pyodbc-5.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8488c3818f12207650836c5c6f7352f9ff9f56a05a05512145995e497c0bbb1"},
+    {file = "pyodbc-5.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0df69e3a500791b70b5748c68a79483b24428e4c16027b56aa0305e95c143a4"},
+    {file = "pyodbc-5.1.0-cp312-cp312-win32.whl", hash = "sha256:aa4e02d3a9bf819394510b726b25f1566f8b3f0891ca400ad2d4c8b86b535b78"},
+    {file = "pyodbc-5.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:33f4984af38872e7bdec78007a34e4d43ae72bf9d0bae3344e79d9d0db157c0e"},
+    {file = "pyodbc-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:29425e2d366e7f5828b76c7993f412a3db4f18bd5bcee00186c00b5a5965e205"},
+    {file = "pyodbc-5.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a2bbd2e75c77dee9f3cd100c3246110abaeb9af3f7fa304ccc2934ff9c6a4fa4"},
+    {file = "pyodbc-5.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3602136a936bc0c1bb9722eb2fbf2042b3ff1ddccdc4688e514b82d4b831563b"},
+    {file = "pyodbc-5.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bed1c843565d3a4fd8c332ebceaf33efe817657a0505eacb97dd1b786a985b0b"},
+    {file = "pyodbc-5.1.0-cp38-cp38-win32.whl", hash = "sha256:735f6da3762e5856b5580be0ed96bb946948346ebd1e526d5169a5513626a67a"},
+    {file = "pyodbc-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:c5bb4e43f6c72f5fa2c634570e0d761767d8ea49f39205229b812fb4d3fe05aa"},
+    {file = "pyodbc-5.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:33f0f1d7764cefef6f787936bd6359670828a6086be67518ab951f1f7f503cda"},
+    {file = "pyodbc-5.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:be3b1c36c31ec7d73d0b34a8ad8743573763fadd8f2bceef1e84408252b48dce"},
+    {file = "pyodbc-5.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e71a51c252b503b4d753e21ed31e640015fc0d00202d42ea42f2396fcc924b4a"},
+    {file = "pyodbc-5.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af5282cc8b667af97d76f4955250619a53f25486cbb6b1f45a06b781006ffa0b"},
+    {file = "pyodbc-5.1.0-cp39-cp39-win32.whl", hash = "sha256:96b2a8dc27693a517e3aad3944a7faa8be95d40d7ec1eda51a1885162eedfa33"},
+    {file = "pyodbc-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:e738c5eedb4a0cbab20cc008882f49b106054499db56864057c2530ff208cf32"},
+    {file = "pyodbc-5.1.0.tar.gz", hash = "sha256:397feee44561a6580be08cedbe986436859563f4bb378f48224655c8e987ea60"},
 ]
 
 [[package]]
@@ -5301,4 +5319,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8,<3.11"
-content-hash = "a65ce99c3f677c9c669cf7bbd385cd6c5980b47dcfa427739ba0d9c4e14f8120"
+content-hash = "d83c04099c45314816923e922b49124679a0802987613a06cf4f24379de7fde4"
diff --git a/pyproject.toml b/pyproject.toml
index cc91f9309c..884ed8d45b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -85,6 +85,7 @@ werkzeug = "2.3.8"
 wtforms = "2.2.1"
 xlsxwriter = "1.2.2"
 tzlocal = "4.3.1"
+pyodbc = "5.1.0"
 
 [tool.poetry.group.all_ds]
 optional = true
@@ -123,7 +124,7 @@ pyhive = "0.6.1"
 pyignite = "0.6.1"
 pymongo = { version = "4.6.3", extras = ["srv", "tls"] }
 pymssql = "2.2.8"
-pyodbc = "4.0.28"
+pyodbc = "5.1.0"
 python-arango = "6.1.0"
 python-rapidjson = "1.1.0"
 requests-aws-sign = "0.1.5"

From c874eb6b11927481c34cf8ee9bfbf60957b602e8 Mon Sep 17 00:00:00 2001
From: Eric Radman <eradman@starfishstorage.com>
Date: Tue, 14 May 2024 22:06:45 -0400
Subject: [PATCH 09/12] Revert changes to job status (#6969)

"Query in queue" should switch to "Executing query", but does not.

Commands:

git revert --no-commit bd17662005248c087a8d6aeadb866ae94d3d3465
git revert --no-commit 5ac5d86f5e4ae0adb0c296e53f95cb79e612237a
vim tests/handlers/test_query_results.py
git add tests/handlers/test_query_results.py

Co-authored-by: Justin Clift <justin@postgresql.org>
---
 .../dashboard-widget/VisualizationWidget.jsx  |  5 +-
 client/app/pages/queries/QuerySource.jsx      |  4 +-
 client/app/pages/queries/QueryView.jsx        |  2 +-
 .../components/QueryExecutionStatus.jsx       | 26 +++---
 client/app/services/query-result.js           | 64 +++++++--------
 client/app/services/query.js                  |  5 +-
 redash/handlers/api.py                        |  6 +-
 redash/handlers/query_results.py              | 17 ++--
 redash/serializers/__init__.py                | 48 ++++++-----
 redash/tasks/queries/execution.py             | 14 ++--
 redash/tasks/worker.py                        |  7 +-
 tests/handlers/test_query_results.py          | 10 ++-
 tests/serializers/test_job.py                 | 81 -------------------
 tests/tasks/test_queries.py                   |  5 +-
 14 files changed, 106 insertions(+), 188 deletions(-)
 delete mode 100644 tests/serializers/test_job.py

diff --git a/client/app/components/dashboards/dashboard-widget/VisualizationWidget.jsx b/client/app/components/dashboards/dashboard-widget/VisualizationWidget.jsx
index 58657e4c6d..9a021cc8bd 100644
--- a/client/app/components/dashboards/dashboard-widget/VisualizationWidget.jsx
+++ b/client/app/components/dashboards/dashboard-widget/VisualizationWidget.jsx
@@ -19,7 +19,6 @@ import PlainButton from "@/components/PlainButton";
 import ExpandedWidgetDialog from "@/components/dashboards/ExpandedWidgetDialog";
 import EditParameterMappingsDialog from "@/components/dashboards/EditParameterMappingsDialog";
 import VisualizationRenderer from "@/components/visualizations/VisualizationRenderer";
-import { ExecutionStatus } from "@/services/query-result";
 
 import Widget from "./Widget";
 
@@ -279,7 +278,7 @@ class VisualizationWidget extends React.Component {
     const widgetQueryResult = widget.getQueryResult();
     const widgetStatus = widgetQueryResult && widgetQueryResult.getStatus();
     switch (widgetStatus) {
-      case ExecutionStatus.FAILED:
+      case "failed":
         return (
           <div className="body-row-auto scrollbox">
             {widgetQueryResult.getError() && (
@@ -289,7 +288,7 @@ class VisualizationWidget extends React.Component {
             )}
           </div>
         );
-      case ExecutionStatus.FINISHED:
+      case "done":
         return (
           <div className="body-row-auto scrollbox">
             <VisualizationRenderer
diff --git a/client/app/pages/queries/QuerySource.jsx b/client/app/pages/queries/QuerySource.jsx
index adb1a0bc4e..f492b99324 100644
--- a/client/app/pages/queries/QuerySource.jsx
+++ b/client/app/pages/queries/QuerySource.jsx
@@ -380,9 +380,7 @@ function QuerySource(props) {
                     <QueryVisualizationTabs
                       queryResult={queryResult}
                       visualizations={query.visualizations}
-                      showNewVisualizationButton={
-                        queryFlags.canEdit && queryResultData.status === ExecutionStatus.FINISHED
-                      }
+                      showNewVisualizationButton={queryFlags.canEdit && queryResultData.status === ExecutionStatus.DONE}
                       canDeleteVisualizations={queryFlags.canEdit}
                       selectedTab={selectedVisualization}
                       onChangeTab={setSelectedVisualization}
diff --git a/client/app/pages/queries/QueryView.jsx b/client/app/pages/queries/QueryView.jsx
index cf46e187f0..cbd582b480 100644
--- a/client/app/pages/queries/QueryView.jsx
+++ b/client/app/pages/queries/QueryView.jsx
@@ -165,7 +165,7 @@ function QueryView(props) {
             <QueryVisualizationTabs
               queryResult={queryResult}
               visualizations={query.visualizations}
-              showNewVisualizationButton={queryFlags.canEdit && queryResultData.status === ExecutionStatus.FINISHED}
+              showNewVisualizationButton={queryFlags.canEdit && queryResultData.status === ExecutionStatus.DONE}
               canDeleteVisualizations={queryFlags.canEdit}
               selectedTab={selectedVisualization}
               onChangeTab={setSelectedVisualization}
diff --git a/client/app/pages/queries/components/QueryExecutionStatus.jsx b/client/app/pages/queries/components/QueryExecutionStatus.jsx
index cd9c444a48..741c85ee52 100644
--- a/client/app/pages/queries/components/QueryExecutionStatus.jsx
+++ b/client/app/pages/queries/components/QueryExecutionStatus.jsx
@@ -1,45 +1,37 @@
+import { includes } from "lodash";
 import React from "react";
 import PropTypes from "prop-types";
 import Alert from "antd/lib/alert";
 import Button from "antd/lib/button";
 import Timer from "@/components/Timer";
-import { ExecutionStatus } from "@/services/query-result";
 
 export default function QueryExecutionStatus({ status, updatedAt, error, isCancelling, onCancel }) {
-  const alertType = status === ExecutionStatus.FAILED ? "error" : "info";
-  const showTimer = status !== ExecutionStatus.FAILED && updatedAt;
-  const isCancelButtonAvailable = [
-    ExecutionStatus.SCHEDULED,
-    ExecutionStatus.QUEUED,
-    ExecutionStatus.STARTED,
-    ExecutionStatus.DEFERRED,
-  ].includes(status);
+  const alertType = status === "failed" ? "error" : "info";
+  const showTimer = status !== "failed" && updatedAt;
+  const isCancelButtonAvailable = includes(["waiting", "processing"], status);
   let message = isCancelling ? <React.Fragment>Cancelling&hellip;</React.Fragment> : null;
 
   switch (status) {
-    case ExecutionStatus.QUEUED:
+    case "waiting":
       if (!isCancelling) {
         message = <React.Fragment>Query in queue&hellip;</React.Fragment>;
       }
       break;
-    case ExecutionStatus.STARTED:
+    case "processing":
       if (!isCancelling) {
         message = <React.Fragment>Executing query&hellip;</React.Fragment>;
       }
       break;
-    case ExecutionStatus.LOADING_RESULT:
+    case "loading-result":
       message = <React.Fragment>Loading results&hellip;</React.Fragment>;
       break;
-    case ExecutionStatus.FAILED:
+    case "failed":
       message = (
         <React.Fragment>
           Error running query: <strong>{error}</strong>
         </React.Fragment>
       );
       break;
-    case ExecutionStatus.CANCELED:
-      message = <React.Fragment>Query was canceled</React.Fragment>;
-      break;
     // no default
   }
 
@@ -74,7 +66,7 @@ QueryExecutionStatus.propTypes = {
 };
 
 QueryExecutionStatus.defaultProps = {
-  status: ExecutionStatus.QUEUED,
+  status: "waiting",
   updatedAt: null,
   error: null,
   isCancelling: true,
diff --git a/client/app/services/query-result.js b/client/app/services/query-result.js
index 6fb10fffa4..7f50228fa6 100644
--- a/client/app/services/query-result.js
+++ b/client/app/services/query-result.js
@@ -50,15 +50,18 @@ const QueryResultResource = {
 };
 
 export const ExecutionStatus = {
-  QUEUED: "queued",
-  STARTED: "started",
-  FINISHED: "finished",
+  WAITING: "waiting",
+  PROCESSING: "processing",
+  DONE: "done",
   FAILED: "failed",
   LOADING_RESULT: "loading-result",
-  CANCELED: "canceled",
-  DEFERRED: "deferred",
-  SCHEDULED: "scheduled",
-  STOPPED: "stopped",
+};
+
+const statuses = {
+  1: ExecutionStatus.WAITING,
+  2: ExecutionStatus.PROCESSING,
+  3: ExecutionStatus.DONE,
+  4: ExecutionStatus.FAILED,
 };
 
 function handleErrorResponse(queryResult, error) {
@@ -77,7 +80,7 @@ function handleErrorResponse(queryResult, error) {
       queryResult.update({
         job: {
           error: "cached query result unavailable, please execute again.",
-          status: ExecutionStatus.FAILED,
+          status: 4,
         },
       });
       return;
@@ -88,7 +91,7 @@ function handleErrorResponse(queryResult, error) {
   queryResult.update({
     job: {
       error: get(error, "response.data.message", "Unknown error occurred. Please try again later."),
-      status: ExecutionStatus.FAILED,
+      status: 4,
     },
   });
 }
@@ -99,19 +102,11 @@ function sleep(ms) {
 
 export function fetchDataFromJob(jobId, interval = 1000) {
   return axios.get(`api/jobs/${jobId}`).then(data => {
-    const status = data.job.status;
-    if (
-      [ExecutionStatus.QUEUED, ExecutionStatus.STARTED, ExecutionStatus.SCHEDULED, ExecutionStatus.DEFERRED].includes(
-        status
-      )
-    ) {
+    const status = statuses[data.job.status];
+    if (status === ExecutionStatus.WAITING || status === ExecutionStatus.PROCESSING) {
       return sleep(interval).then(() => fetchDataFromJob(data.job.id));
-    } else if (status === ExecutionStatus.FINISHED) {
-      return data.job.result_id;
-    } else if (status === ExecutionStatus.CANCELED) {
-      return Promise.reject("Job was canceled");
-    } else if (status === ExecutionStatus.STOPPED) {
-      return Promise.reject("Job was stopped");
+    } else if (status === ExecutionStatus.DONE) {
+      return data.job.result;
     } else if (status === ExecutionStatus.FAILED) {
       return Promise.reject(data.job.error);
     }
@@ -127,7 +122,7 @@ class QueryResult {
     this.deferred = defer();
     this.job = {};
     this.query_result = {};
-    this.status = ExecutionStatus.QUEUED;
+    this.status = "waiting";
 
     this.updatedAt = moment();
 
@@ -143,8 +138,8 @@ class QueryResult {
     extend(this, props);
 
     if ("query_result" in props) {
-      this.status = ExecutionStatus.FINISHED;
-      this.deferred.onStatusChange(ExecutionStatus.FINISHED);
+      this.status = ExecutionStatus.DONE;
+      this.deferred.onStatusChange(ExecutionStatus.DONE);
 
       const columnTypes = {};
 
@@ -188,10 +183,11 @@ class QueryResult {
       });
 
       this.deferred.resolve(this);
-    } else if (this.job.status === ExecutionStatus.STARTED || this.job.status === ExecutionStatus.FINISHED) {
-      this.status = ExecutionStatus.STARTED;
-    } else if (this.job.status === ExecutionStatus.FAILED) {
-      this.status = this.job.status;
+    } else if (this.job.status === 3 || this.job.status === 2) {
+      this.deferred.onStatusChange(ExecutionStatus.PROCESSING);
+      this.status = "processing";
+    } else if (this.job.status === 4) {
+      this.status = statuses[this.job.status];
       this.deferred.reject(new QueryResultError(this.job.error));
     } else {
       this.deferred.onStatusChange(undefined);
@@ -215,7 +211,7 @@ class QueryResult {
     if (this.isLoadingResult) {
       return ExecutionStatus.LOADING_RESULT;
     }
-    return this.status || this.job.status;
+    return this.status || statuses[this.job.status];
   }
 
   getError() {
@@ -378,7 +374,7 @@ class QueryResult {
     this.isLoadingResult = true;
     this.deferred.onStatusChange(ExecutionStatus.LOADING_RESULT);
 
-    QueryResultResource.get({ id: this.job.result_id })
+    QueryResultResource.get({ id: this.job.query_result_id })
       .then(response => {
         this.update(response);
         this.isLoadingResult = false;
@@ -393,7 +389,7 @@ class QueryResult {
           this.update({
             job: {
               error: "failed communicating with server. Please check your Internet connection and try again.",
-              status: ExecutionStatus.FAILED,
+              status: 4,
             },
           });
           this.isLoadingResult = false;
@@ -417,9 +413,9 @@ class QueryResult {
       .then(jobResponse => {
         this.update(jobResponse);
 
-        if (this.getStatus() === ExecutionStatus.STARTED && this.job.result_id && this.job.result_id !== "None") {
+        if (this.getStatus() === "processing" && this.job.query_result_id && this.job.query_result_id !== "None") {
           loadResult();
-        } else if (this.getStatus() !== ExecutionStatus.FAILED) {
+        } else if (this.getStatus() !== "failed") {
           const waitTime = tryNumber > 10 ? 3000 : 500;
           setTimeout(() => {
             this.refreshStatus(query, parameters, tryNumber + 1);
@@ -432,7 +428,7 @@ class QueryResult {
         this.update({
           job: {
             error: "failed communicating with server. Please check your Internet connection and try again.",
-            status: ExecutionStatus.FAILED,
+            status: 4,
           },
         });
       });
diff --git a/client/app/services/query.js b/client/app/services/query.js
index 76c8cf858a..a8cf624cb8 100644
--- a/client/app/services/query.js
+++ b/client/app/services/query.js
@@ -2,7 +2,6 @@ import moment from "moment";
 import debug from "debug";
 import Mustache from "mustache";
 import { axios } from "@/services/axios";
-import { ExecutionStatus } from "@/services/query-result";
 import {
   zipObject,
   isEmpty,
@@ -104,7 +103,7 @@ export class Query {
       return new QueryResult({
         job: {
           error: `missing ${valuesWord} for ${missingParams.join(", ")} ${paramsWord}.`,
-          status: ExecutionStatus.FAILED,
+          status: 4,
         },
       });
     }
@@ -361,7 +360,7 @@ export class QueryResultError {
 
   // eslint-disable-next-line class-methods-use-this
   getStatus() {
-    return ExecutionStatus.FAILED;
+    return "failed";
   }
 
   // eslint-disable-next-line class-methods-use-this
diff --git a/redash/handlers/api.py b/redash/handlers/api.py
index 76de2f5a2d..48428daf0f 100644
--- a/redash/handlers/api.py
+++ b/redash/handlers/api.py
@@ -236,11 +236,11 @@ def json_representation(data, code, headers=None):
 )
 api.add_org_resource(
     QueryResultResource,
-    "/api/query_results/<result_id>.<filetype>",
-    "/api/query_results/<result_id>",
+    "/api/query_results/<query_result_id>.<filetype>",
+    "/api/query_results/<query_result_id>",
     "/api/queries/<query_id>/results",
     "/api/queries/<query_id>/results.<filetype>",
-    "/api/queries/<query_id>/results/<result_id>.<filetype>",
+    "/api/queries/<query_id>/results/<query_result_id>.<filetype>",
     endpoint="query_result",
 )
 api.add_org_resource(
diff --git a/redash/handlers/query_results.py b/redash/handlers/query_results.py
index 22a7a8631e..bfc4371d08 100644
--- a/redash/handlers/query_results.py
+++ b/redash/handlers/query_results.py
@@ -5,7 +5,6 @@
 from flask import make_response, request
 from flask_login import current_user
 from flask_restful import abort
-from rq.job import JobStatus
 
 from redash import models, settings
 from redash.handlers.base import BaseResource, get_object_or_404, record_event
@@ -39,7 +38,7 @@
 
 
 def error_response(message, http_status=400):
-    return {"job": {"status": JobStatus.FAILED, "error": message}}, http_status
+    return {"job": {"status": 4, "error": message}}, http_status
 
 
 error_messages = {
@@ -226,7 +225,7 @@ def add_cors_headers(headers):
                 headers["Access-Control-Allow-Credentials"] = str(settings.ACCESS_CONTROL_ALLOW_CREDENTIALS).lower()
 
     @require_any_of_permission(("view_query", "execute_query"))
-    def options(self, query_id=None, result_id=None, filetype="json"):
+    def options(self, query_id=None, query_result_id=None, filetype="json"):
         headers = {}
         self.add_cors_headers(headers)
 
@@ -286,12 +285,12 @@ def post(self, query_id):
                 return error_messages["no_permission"]
 
     @require_any_of_permission(("view_query", "execute_query"))
-    def get(self, query_id=None, result_id=None, filetype="json"):
+    def get(self, query_id=None, query_result_id=None, filetype="json"):
         """
         Retrieve query results.
 
         :param number query_id: The ID of the query whose results should be fetched
-        :param number result_id: the ID of the query result to fetch
+        :param number query_result_id: the ID of the query result to fetch
         :param string filetype: Format to return. One of 'json', 'xlsx', or 'csv'. Defaults to 'json'.
 
         :<json number id: Query result ID
@@ -306,13 +305,13 @@ def get(self, query_id=None, result_id=None, filetype="json"):
         # This method handles two cases: retrieving result by id & retrieving result by query id.
         # They need to be split, as they have different logic (for example, retrieving by query id
         # should check for query parameters and shouldn't cache the result).
-        should_cache = result_id is not None
+        should_cache = query_result_id is not None
 
         query_result = None
         query = None
 
-        if result_id:
-            query_result = get_object_or_404(models.QueryResult.get_by_id_and_org, result_id, self.current_org)
+        if query_result_id:
+            query_result = get_object_or_404(models.QueryResult.get_by_id_and_org, query_result_id, self.current_org)
 
         if query_id is not None:
             query = get_object_or_404(models.Query.get_by_id_and_org, query_id, self.current_org)
@@ -347,7 +346,7 @@ def get(self, query_id=None, result_id=None, filetype="json"):
                     event["object_id"] = query_id
                 else:
                     event["object_type"] = "query_result"
-                    event["object_id"] = result_id
+                    event["object_id"] = query_result_id
 
                 self.record_event(event)
 
diff --git a/redash/serializers/__init__.py b/redash/serializers/__init__.py
index 3919db6ea2..41a370e43e 100644
--- a/redash/serializers/__init__.py
+++ b/redash/serializers/__init__.py
@@ -7,7 +7,6 @@
 from flask_login import current_user
 from funcy import project
 from rq.job import JobStatus
-from rq.results import Result
 from rq.timeouts import JobTimeoutException
 
 from redash import models
@@ -272,33 +271,46 @@ def serialize(self):
 
 
 def serialize_job(job):
+    # TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
+    STATUSES = {
+        JobStatus.QUEUED: 1,
+        JobStatus.STARTED: 2,
+        JobStatus.FINISHED: 3,
+        JobStatus.FAILED: 4,
+        JobStatus.CANCELED: 5,
+        JobStatus.DEFERRED: 6,
+        JobStatus.SCHEDULED: 7,
+    }
+
+    job_status = job.get_status()
     if job.is_started:
         updated_at = job.started_at or 0
     else:
         updated_at = 0
 
-    status = job.get_status()
-    error = result_id = None
-    job_result = job.latest_result()
-    if job_result:
-        if job_result.type == Result.Type.SUCCESSFUL:
-            result = job_result.return_value
-            if isinstance(result, Exception):
-                error = str(result)
-                status = JobStatus.FAILED
-            elif isinstance(result, dict) and "error" in result:
-                error = result["error"]
-                status = JobStatus.FAILED
-            else:
-                result_id = result
-        else:
-            error = job_result.exc_string
+    status = STATUSES[job_status]
+    result = query_result_id = None
+
+    if job.is_cancelled:
+        error = "Query cancelled by user."
+        status = 4
+    elif isinstance(job.result, Exception):
+        error = str(job.result)
+        status = 4
+    elif isinstance(job.result, dict) and "error" in job.result:
+        error = job.result["error"]
+        status = 4
+    else:
+        error = ""
+        result = query_result_id = job.result
+
     return {
         "job": {
             "id": job.id,
             "updated_at": updated_at,
             "status": status,
             "error": error,
-            "result_id": result_id,
+            "result": result,
+            "query_result_id": query_result_id,
         }
     }
diff --git a/redash/tasks/queries/execution.py b/redash/tasks/queries/execution.py
index 90b44ff96e..695375f99c 100644
--- a/redash/tasks/queries/execution.py
+++ b/redash/tasks/queries/execution.py
@@ -43,24 +43,24 @@ def enqueue_query(query, data_source, user_id, is_api_key=False, scheduled_query
             if job_id:
                 logger.info("[%s] Found existing job: %s", query_hash, job_id)
                 job_complete = None
+                job_cancelled = None
 
                 try:
                     job = Job.fetch(job_id)
                     job_exists = True
                     status = job.get_status()
-                    job_complete = status in [
-                        JobStatus.FINISHED,
-                        JobStatus.FAILED,
-                        JobStatus.STOPPED,
-                        JobStatus.CANCELED,
-                    ]
+                    job_complete = status in [JobStatus.FINISHED, JobStatus.FAILED]
+                    job_cancelled = job.is_cancelled
+
                     if job_complete:
                         message = "job found is complete (%s)" % status
+                    elif job_cancelled:
+                        message = "job found has ben cancelled"
                 except NoSuchJobError:
                     message = "job found has expired"
                     job_exists = False
 
-                lock_is_irrelevant = job_complete or not job_exists
+                lock_is_irrelevant = job_complete or job_cancelled or not job_exists
 
                 if lock_is_irrelevant:
                     logger.info("[%s] %s, removing lock", query_hash, message)
diff --git a/redash/tasks/worker.py b/redash/tasks/worker.py
index e1bfd457db..1983168d3d 100644
--- a/redash/tasks/worker.py
+++ b/redash/tasks/worker.py
@@ -65,7 +65,10 @@ def execute_job(self, job, queue):
             super().execute_job(job, queue)
         finally:
             statsd_client.decr("rq.jobs.running.{}".format(queue.name))
-            statsd_client.incr("rq.jobs.{}.{}".format(job.get_status(), queue.name))
+            if job.get_status() == JobStatus.FINISHED:
+                statsd_client.incr("rq.jobs.finished.{}".format(queue.name))
+            else:
+                statsd_client.incr("rq.jobs.failed.{}".format(queue.name))
 
 
 class HardLimitingWorker(BaseWorker):
@@ -151,7 +154,7 @@ def monitor_work_horse(self, job, queue):
         job_status = job.get_status()
         if job_status is None:  # Job completed and its ttl has expired
             return
-        if job_status not in [JobStatus.FINISHED, JobStatus.FAILED, JobStatus.STOPPED, JobStatus.CANCELED]:
+        if job_status not in [JobStatus.FINISHED, JobStatus.FAILED]:
             if not job.ended_at:
                 job.ended_at = utcnow()
 
diff --git a/tests/handlers/test_query_results.py b/tests/handlers/test_query_results.py
index cbecafee62..1170d6941e 100644
--- a/tests/handlers/test_query_results.py
+++ b/tests/handlers/test_query_results.py
@@ -1,5 +1,3 @@
-from rq.job import JobStatus
-
 from redash.handlers.query_results import error_messages, run_query
 from redash.models import db
 from tests import BaseTestCase
@@ -436,6 +434,9 @@ def test_renders_excel_file_when_rows_have_missing_columns(self):
 
 class TestJobResource(BaseTestCase):
     def test_cancels_queued_queries(self):
+        QUEUED = 1
+        FAILED = 4
+
         query = self.factory.create_query()
         job_id = self.make_request(
             "post",
@@ -446,9 +447,10 @@ def test_cancels_queued_queries(self):
         ]["id"]
 
         status = self.make_request("get", f"/api/jobs/{job_id}").json["job"]["status"]
-        self.assertEqual(status, JobStatus.QUEUED)
+        self.assertEqual(status, QUEUED)
 
         self.make_request("delete", f"/api/jobs/{job_id}")
 
         job = self.make_request("get", f"/api/jobs/{job_id}").json["job"]
-        self.assertEqual(job["status"], JobStatus.CANCELED)
+        self.assertEqual(job["status"], FAILED)
+        self.assertTrue("cancelled" in job["error"])
diff --git a/tests/serializers/test_job.py b/tests/serializers/test_job.py
deleted file mode 100644
index bcb044ee09..0000000000
--- a/tests/serializers/test_job.py
+++ /dev/null
@@ -1,81 +0,0 @@
-from unittest.mock import MagicMock
-
-from rq.job import JobStatus
-from rq.results import Result
-
-from redash.serializers import (
-    serialize_job,
-)
-from redash.tasks.queries.execution import QueryExecutionError
-from tests import BaseTestCase
-
-
-class JobSerializationTest(BaseTestCase):
-    def test_serializes_job_with_exception_in_result(self):
-        job = MagicMock()
-        job.id = 0
-        job.is_started = False
-        job.get_status = MagicMock(return_value=JobStatus.FINISHED)
-        result = MagicMock()
-        result.type = Result.Type.SUCCESSFUL
-        result.return_value = QueryExecutionError("test")
-        job.latest_result = MagicMock(return_value=result)
-        result = serialize_job(job)
-        self.assertDictEqual(
-            result,
-            {
-                "job": {
-                    "id": 0,
-                    "updated_at": 0,
-                    "status": JobStatus.FAILED,
-                    "error": str(QueryExecutionError("test")),
-                    "result_id": None,
-                }
-            },
-        )
-
-    def test_serializes_job_with_dict_that_contains_error_in_result(self):
-        job = MagicMock()
-        job.id = 0
-        job.is_started = False
-        job.get_status = MagicMock(return_value=JobStatus.FINISHED)
-        result = MagicMock()
-        result.type = Result.Type.SUCCESSFUL
-        result.return_value = {"error": "test error"}
-        job.latest_result = MagicMock(return_value=result)
-        result = serialize_job(job)
-        self.assertDictEqual(
-            result,
-            {
-                "job": {
-                    "id": 0,
-                    "updated_at": 0,
-                    "status": JobStatus.FAILED,
-                    "error": "test error",
-                    "result_id": None,
-                }
-            },
-        )
-
-    def test_serializes_job_with_dict_that_finished_successfully(self):
-        job = MagicMock()
-        job.id = 0
-        job.is_started = False
-        job.get_status = MagicMock(return_value=JobStatus.FINISHED)
-        result = MagicMock()
-        result.type = Result.Type.SUCCESSFUL
-        result.return_value = 1
-        job.latest_result = MagicMock(return_value=result)
-        result = serialize_job(job)
-        self.assertDictEqual(
-            result,
-            {
-                "job": {
-                    "id": 0,
-                    "updated_at": 0,
-                    "status": JobStatus.FINISHED,
-                    "error": None,
-                    "result_id": 1,
-                }
-            },
-        )
diff --git a/tests/tasks/test_queries.py b/tests/tasks/test_queries.py
index 9ca2ec7c86..93d5d73b86 100644
--- a/tests/tasks/test_queries.py
+++ b/tests/tasks/test_queries.py
@@ -1,7 +1,6 @@
 from mock import Mock, patch
 from rq import Connection
 from rq.exceptions import NoSuchJobError
-from rq.job import JobStatus
 
 from redash import models, rq_redis_connection
 from redash.query_runner.pg import PostgreSQL
@@ -22,7 +21,7 @@ def fetch_job(*args, **kwargs):
 
     result = Mock()
     result.id = job_id
-    result.get_status = lambda: JobStatus.STARTED
+    result.is_cancelled = False
 
     return result
 
@@ -108,7 +107,7 @@ def test_reenqueue_during_job_cancellation(self, enqueue, my_fetch_job):
             # "cancel" the previous job
             def cancel_job(*args, **kwargs):
                 job = fetch_job(*args, **kwargs)
-                job.get_status = lambda: JobStatus.CANCELED
+                job.is_cancelled = True
                 return job
 
             my_fetch_job.side_effect = cancel_job

From 10a46fd33c86146eef0bb388b8002330fc829981 Mon Sep 17 00:00:00 2001
From: Eric Radman <eradman@starfishstorage.com>
Date: Wed, 15 May 2024 23:28:42 -0400
Subject: [PATCH 10/12] Revert "show pg and athena column comments and table
 descriptions as antd tooltip if they are defined (#6582)" (#6971)

This reverts commit c12d45077a992e9d5d21bc891d4961c9a2ff73f6.

This commit did not sort tables properly by schema, then name
---
 .../app/components/queries/SchemaBrowser.jsx  | 74 ++++++-------------
 redash/models/__init__.py                     | 11 +--
 redash/query_runner/athena.py                 | 39 ++--------
 redash/query_runner/pg.py                     | 18 +----
 tests/models/test_data_sources.py             | 20 ++---
 tests/query_runner/test_athena.py             | 23 +-----
 6 files changed, 43 insertions(+), 142 deletions(-)

diff --git a/client/app/components/queries/SchemaBrowser.jsx b/client/app/components/queries/SchemaBrowser.jsx
index 730f4acf88..ed8d2402ed 100644
--- a/client/app/components/queries/SchemaBrowser.jsx
+++ b/client/app/components/queries/SchemaBrowser.jsx
@@ -16,7 +16,6 @@ import LoadingState from "../items-list/components/LoadingState";
 const SchemaItemColumnType = PropTypes.shape({
   name: PropTypes.string.isRequired,
   type: PropTypes.string,
-  comment: PropTypes.string,
 });
 
 export const SchemaItemType = PropTypes.shape({
@@ -48,30 +47,13 @@ function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
   return (
     <div {...props}>
       <div className="schema-list-item">
-        {item.description ? (
-          <Tooltip
-            title={item.description}
-            mouseEnterDelay={0}
-            mouseLeaveDelay={0}
-            placement="right"
-            arrowPointAtCenter>
-            <PlainButton className="table-name" onClick={onToggle}>
-              <i className="fa fa-table m-r-5" aria-hidden="true" />
-              <strong>
-                <span title={item.name}>{tableDisplayName}</span>
-                {!isNil(item.size) && <span> ({item.size})</span>}
-              </strong>
-            </PlainButton>
-          </Tooltip>
-        ) : (
-          <PlainButton className="table-name" onClick={onToggle}>
-            <i className="fa fa-table m-r-5" aria-hidden="true" />
-            <strong>
-              <span title={item.name}>{tableDisplayName}</span>
-              {!isNil(item.size) && <span> ({item.size})</span>}
-            </strong>
-          </PlainButton>
-        )}
+        <PlainButton className="table-name" onClick={onToggle}>
+          <i className="fa fa-table m-r-5" aria-hidden="true" />
+          <strong>
+            <span title={item.name}>{tableDisplayName}</span>
+            {!isNil(item.size) && <span> ({item.size})</span>}
+          </strong>
+        </PlainButton>
         <Tooltip
           title="Insert table name into query text"
           mouseEnterDelay={0}
@@ -91,34 +73,22 @@ function SchemaItem({ item, expanded, onToggle, onSelect, ...props }) {
             map(item.columns, column => {
               const columnName = get(column, "name");
               const columnType = get(column, "type");
-              const columnComment = get(column, "comment");
-              if (columnComment) {
-                return (
-                  <Tooltip title={columnComment} mouseEnterDelay={0} mouseLeaveDelay={0} placement="rightTop">
-                    <PlainButton
-                      key={columnName}
-                      className="table-open-item"
-                      onClick={e => handleSelect(e, columnName)}>
-                      <div>
-                        {columnName} {columnType && <span className="column-type">{columnType}</span>}
-                      </div>
-
-                      <div className="copy-to-editor">
-                        <i className="fa fa-angle-double-right" aria-hidden="true" />
-                      </div>
-                    </PlainButton>
-                  </Tooltip>
-                );
-              }
               return (
-                <PlainButton key={columnName} className="table-open-item" onClick={e => handleSelect(e, columnName)}>
-                  <div>
-                    {columnName} {columnType && <span className="column-type">{columnType}</span>}
-                  </div>
-                  <div className="copy-to-editor">
-                    <i className="fa fa-angle-double-right" aria-hidden="true" />
-                  </div>
-                </PlainButton>
+                <Tooltip
+                  title="Insert column name into query text"
+                  mouseEnterDelay={0}
+                  mouseLeaveDelay={0}
+                  placement="rightTop">
+                  <PlainButton key={columnName} className="table-open-item" onClick={e => handleSelect(e, columnName)}>
+                    <div>
+                      {columnName} {columnType && <span className="column-type">{columnType}</span>}
+                    </div>
+
+                    <div className="copy-to-editor">
+                      <i className="fa fa-angle-double-right" aria-hidden="true" />
+                    </div>
+                  </PlainButton>
+                </Tooltip>
               );
             })
           )}
diff --git a/redash/models/__init__.py b/redash/models/__init__.py
index 7ee0d99741..91ddd91ee3 100644
--- a/redash/models/__init__.py
+++ b/redash/models/__init__.py
@@ -227,16 +227,7 @@ def get_schema(self, refresh=False):
 
     def _sort_schema(self, schema):
         return [
-            {
-                "name": i["name"],
-                "description": i.get("description"),
-                "columns": sorted(
-                    i["columns"],
-                    key=lambda col: (
-                        ("partition" in col["type"], col.get("idx", 0), col["name"]) if isinstance(col, dict) else col
-                    ),
-                ),
-            }
+            {"name": i["name"], "columns": sorted(i["columns"], key=lambda x: x["name"] if isinstance(x, dict) else x)}
             for i in sorted(schema, key=lambda x: x["name"])
         ]
 
diff --git a/redash/query_runner/athena.py b/redash/query_runner/athena.py
index 0a5f648417..acde734f02 100644
--- a/redash/query_runner/athena.py
+++ b/redash/query_runner/athena.py
@@ -21,9 +21,7 @@
 
 try:
     import boto3
-    import pandas as pd
     import pyathena
-    from pyathena.pandas_cursor import PandasCursor
 
     enabled = True
 except ImportError:
@@ -190,35 +188,10 @@ def __get_schema_from_glue(self):
                         logger.warning("Glue table doesn't have StorageDescriptor: %s", table_name)
                         continue
                     if table_name not in schema:
-                        columns = []
-                        for cols in table["StorageDescriptor"]["Columns"]:
-                            c = {
-                                "name": cols["Name"],
-                            }
-                            if "Type" in cols:
-                                c["type"] = cols["Type"]
-                            if "Comment" in cols:
-                                c["comment"] = cols["Comment"]
-                            columns.append(c)
-
-                        schema[table_name] = {
-                            "name": table_name,
-                            "columns": columns,
-                            "description": table.get("Description"),
-                        }
-                        for idx, partition in enumerate(table.get("PartitionKeys", [])):
-                            schema[table_name]["columns"].append(
-                                {
-                                    "name": partition["Name"],
-                                    "type": "partition",
-                                    "idx": idx,
-                                }
-                            )
-                            if "Type" in partition:
-                                _type = partition["Type"]
-                                c["type"] = f"partition ({_type})"
-                            if "Comment" in partition:
-                                c["comment"] = partition["Comment"]
+                        column = [columns["Name"] for columns in table["StorageDescriptor"]["Columns"]]
+                        schema[table_name] = {"name": table_name, "columns": column}
+                        for partition in table.get("PartitionKeys", []):
+                            schema[table_name]["columns"].append(partition["Name"])
         return list(schema.values())
 
     def get_schema(self, get_stats=False):
@@ -252,7 +225,6 @@ def run_query(self, query, user):
             kms_key=self.configuration.get("kms_key", None),
             work_group=self.configuration.get("work_group", "primary"),
             formatter=SimpleFormatter(),
-            cursor_class=PandasCursor,
             **self._get_iam_credentials(user=user),
         ).cursor()
 
@@ -260,8 +232,7 @@ def run_query(self, query, user):
             cursor.execute(query)
             column_tuples = [(i[0], _TYPE_MAPPINGS.get(i[1], None)) for i in cursor.description]
             columns = self.fetch_columns(column_tuples)
-            df = cursor.as_pandas().replace({pd.NA: None})
-            rows = df.to_dict(orient="records")
+            rows = [dict(zip(([c["name"] for c in columns]), r)) for i, r in enumerate(cursor.fetchall())]
             qbytes = None
             athena_query_id = None
             try:
diff --git a/redash/query_runner/pg.py b/redash/query_runner/pg.py
index 46bf3cc0b8..dc74aff2cc 100644
--- a/redash/query_runner/pg.py
+++ b/redash/query_runner/pg.py
@@ -108,8 +108,6 @@ def build_schema(query_result, schema):
         column = row["column_name"]
         if row.get("data_type") is not None:
             column = {"name": row["column_name"], "type": row["data_type"]}
-            if "column_comment" in row:
-                column["comment"] = row["column_comment"]
 
         schema[table_name]["columns"].append(column)
 
@@ -224,9 +222,7 @@ def _get_tables(self, schema):
         SELECT s.nspname as table_schema,
                c.relname as table_name,
                a.attname as column_name,
-               null as data_type,
-               null as column_comment,
-               null as idx
+               null as data_type
         FROM pg_class c
         JOIN pg_namespace s
         ON c.relnamespace = s.oid
@@ -242,16 +238,8 @@ def _get_tables(self, schema):
         SELECT table_schema,
                table_name,
                column_name,
-               data_type,
-               pgd.description,
-               isc.ordinal_position
-        FROM information_schema.columns as isc
-        LEFT JOIN pg_catalog.pg_statio_all_tables as st
-        ON isc.table_schema = st.schemaname
-        AND isc.table_name = st.relname
-        LEFT JOIN pg_catalog.pg_description pgd
-        ON pgd.objoid=st.relid
-        AND pgd.objsubid=isc.ordinal_position
+               data_type
+        FROM information_schema.columns
         WHERE table_schema NOT IN ('pg_catalog', 'information_schema')
         """
 
diff --git a/tests/models/test_data_sources.py b/tests/models/test_data_sources.py
index 82bd4b5a7a..7db4d463a3 100644
--- a/tests/models/test_data_sources.py
+++ b/tests/models/test_data_sources.py
@@ -8,7 +8,7 @@
 
 class DataSourceTest(BaseTestCase):
     def test_get_schema(self):
-        return_value = [{"name": "table", "columns": [], "description": None}]
+        return_value = [{"name": "table", "columns": []}]
 
         with mock.patch("redash.query_runner.pg.PostgreSQL.get_schema") as patched_get_schema:
             patched_get_schema.return_value = return_value
@@ -18,7 +18,7 @@ def test_get_schema(self):
             self.assertEqual(return_value, schema)
 
     def test_get_schema_uses_cache(self):
-        return_value = [{"name": "table", "columns": [], "description": None}]
+        return_value = [{"name": "table", "columns": []}]
         with mock.patch("redash.query_runner.pg.PostgreSQL.get_schema") as patched_get_schema:
             patched_get_schema.return_value = return_value
 
@@ -29,12 +29,12 @@ def test_get_schema_uses_cache(self):
             self.assertEqual(patched_get_schema.call_count, 1)
 
     def test_get_schema_skips_cache_with_refresh_true(self):
-        return_value = [{"name": "table", "columns": [], "description": None}]
+        return_value = [{"name": "table", "columns": []}]
         with mock.patch("redash.query_runner.pg.PostgreSQL.get_schema") as patched_get_schema:
             patched_get_schema.return_value = return_value
 
             self.factory.data_source.get_schema()
-            new_return_value = [{"name": "new_table", "columns": [], "description": None}]
+            new_return_value = [{"name": "new_table", "columns": []}]
             patched_get_schema.return_value = new_return_value
             schema = self.factory.data_source.get_schema(refresh=True)
 
@@ -43,11 +43,10 @@ def test_get_schema_skips_cache_with_refresh_true(self):
 
     def test_schema_sorter(self):
         input_data = [
-            {"name": "zoo", "columns": ["is_zebra", "is_snake", "is_cow"], "description": None},
+            {"name": "zoo", "columns": ["is_zebra", "is_snake", "is_cow"]},
             {
                 "name": "all_terain_vehicle",
                 "columns": ["has_wheels", "has_engine", "has_all_wheel_drive"],
-                "description": None,
             },
         ]
 
@@ -55,9 +54,8 @@ def test_schema_sorter(self):
             {
                 "name": "all_terain_vehicle",
                 "columns": ["has_all_wheel_drive", "has_engine", "has_wheels"],
-                "description": None,
             },
-            {"name": "zoo", "columns": ["is_cow", "is_snake", "is_zebra"], "description": None},
+            {"name": "zoo", "columns": ["is_cow", "is_snake", "is_zebra"]},
         ]
 
         real_output = self.factory.data_source._sort_schema(input_data)
@@ -66,11 +64,10 @@ def test_schema_sorter(self):
 
     def test_model_uses_schema_sorter(self):
         orig_schema = [
-            {"name": "zoo", "columns": ["is_zebra", "is_snake", "is_cow"], "description": None},
+            {"name": "zoo", "columns": ["is_zebra", "is_snake", "is_cow"]},
             {
                 "name": "all_terain_vehicle",
                 "columns": ["has_wheels", "has_engine", "has_all_wheel_drive"],
-                "description": None,
             },
         ]
 
@@ -78,9 +75,8 @@ def test_model_uses_schema_sorter(self):
             {
                 "name": "all_terain_vehicle",
                 "columns": ["has_all_wheel_drive", "has_engine", "has_wheels"],
-                "description": None,
             },
-            {"name": "zoo", "columns": ["is_cow", "is_snake", "is_zebra"], "description": None},
+            {"name": "zoo", "columns": ["is_cow", "is_snake", "is_zebra"]},
         ]
 
         with mock.patch("redash.query_runner.pg.PostgreSQL.get_schema") as patched_get_schema:
diff --git a/tests/query_runner/test_athena.py b/tests/query_runner/test_athena.py
index 2c386ead11..2ac4ee42fe 100644
--- a/tests/query_runner/test_athena.py
+++ b/tests/query_runner/test_athena.py
@@ -75,9 +75,7 @@ def test_external_table(self):
             {"DatabaseName": "test1"},
         )
         with self.stubber:
-            assert query_runner.get_schema() == [
-                {"columns": [{"name": "row_id", "type": "int"}], "name": "test1.jdbc_table", "description": None}
-            ]
+            assert query_runner.get_schema() == [{"columns": ["row_id"], "name": "test1.jdbc_table"}]
 
     def test_partitioned_table(self):
         """
@@ -126,16 +124,7 @@ def test_partitioned_table(self):
             {"DatabaseName": "test1"},
         )
         with self.stubber:
-            assert query_runner.get_schema() == [
-                {
-                    "columns": [
-                        {"name": "sk", "type": "partition (int)"},
-                        {"name": "category", "type": "partition", "idx": 0},
-                    ],
-                    "name": "test1.partitioned_table",
-                    "description": None,
-                }
-            ]
+            assert query_runner.get_schema() == [{"columns": ["sk", "category"], "name": "test1.partitioned_table"}]
 
     def test_view(self):
         query_runner = Athena({"glue": True, "region": "mars-east-1"})
@@ -167,9 +156,7 @@ def test_view(self):
             {"DatabaseName": "test1"},
         )
         with self.stubber:
-            assert query_runner.get_schema() == [
-                {"columns": [{"name": "sk", "type": "int"}], "name": "test1.view", "description": None}
-            ]
+            assert query_runner.get_schema() == [{"columns": ["sk"], "name": "test1.view"}]
 
     def test_dodgy_table_does_not_break_schema_listing(self):
         """
@@ -209,9 +196,7 @@ def test_dodgy_table_does_not_break_schema_listing(self):
             {"DatabaseName": "test1"},
         )
         with self.stubber:
-            assert query_runner.get_schema() == [
-                {"columns": [{"name": "region", "type": "string"}], "name": "test1.csv", "description": None}
-            ]
+            assert query_runner.get_schema() == [{"columns": ["region"], "name": "test1.csv"}]
 
     def test_no_storage_descriptor_table(self):
         """

From bd4ba96c4310941e6a012f11e14ddeb2ca8e7841 Mon Sep 17 00:00:00 2001
From: Justin Clift <justin@postgresql.org>
Date: Sat, 18 May 2024 18:18:37 +1000
Subject: [PATCH 11/12] Typo fix in message (#6979)

---
 redash/tasks/queries/execution.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/redash/tasks/queries/execution.py b/redash/tasks/queries/execution.py
index 695375f99c..cba2aded07 100644
--- a/redash/tasks/queries/execution.py
+++ b/redash/tasks/queries/execution.py
@@ -55,7 +55,7 @@ def enqueue_query(query, data_source, user_id, is_api_key=False, scheduled_query
                     if job_complete:
                         message = "job found is complete (%s)" % status
                     elif job_cancelled:
-                        message = "job found has ben cancelled"
+                        message = "job found has been cancelled"
                 except NoSuchJobError:
                     message = "job found has expired"
                     job_exists = False

From 60d3c66a8b984a0e494d41ba97c63903762c799d Mon Sep 17 00:00:00 2001
From: Arik Fraimovich <arik.fraimovich@databricks.com>
Date: Sat, 18 May 2024 07:36:29 -0700
Subject: [PATCH 12/12] Merge pull request from GHSA-32fw-wc7f-7qg9

---
 redash/authentication/ldap_auth.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/redash/authentication/ldap_auth.py b/redash/authentication/ldap_auth.py
index 1a19cce05b..3bc5ff272f 100644
--- a/redash/authentication/ldap_auth.py
+++ b/redash/authentication/ldap_auth.py
@@ -8,6 +8,7 @@
 
 try:
     from ldap3 import Connection, Server
+    from ldap3.utils.conv import escape_filter_chars
 except ImportError:
     if settings.LDAP_LOGIN_ENABLED:
         sys.exit(
@@ -69,6 +70,7 @@ def login(org_slug=None):
 
 
 def auth_ldap_user(username, password):
+    clean_username = escape_filter_chars(username)
     server = Server(settings.LDAP_HOST_URL, use_ssl=settings.LDAP_SSL)
     if settings.LDAP_BIND_DN is not None:
         conn = Connection(
@@ -83,7 +85,7 @@ def auth_ldap_user(username, password):
 
     conn.search(
         settings.LDAP_SEARCH_DN,
-        settings.LDAP_SEARCH_TEMPLATE % {"username": username},
+        settings.LDAP_SEARCH_TEMPLATE % {"username": clean_username},
         attributes=[settings.LDAP_DISPLAY_NAME_KEY, settings.LDAP_EMAIL_KEY],
     )