From 9a16d4e21922af60b13463f4fd4c31f45761a5c0 Mon Sep 17 00:00:00 2001
From: Abhishek Radhakrishnan <abhishek.rb19@gmail.com>
Date: Fri, 18 Oct 2024 14:39:01 -0700
Subject: [PATCH 01/12] Move SqlTaskStatus and SqlTaskStausTest from msq module
 to sql module. (#17380)

- This is a non-functional change that moves SqlTaskStatus and its unit test SqlTaskStatusTest from the msq module to the sql module to help class reuse in other places.
- This refactor is extracted from this PR to facilitate easier review.
- Fix a minor spacing issue in the TaskStartTimeoutFault error message.
---
 .../druid/msq/indexing/error/TaskStartTimeoutFault.java      | 2 +-
 .../org/apache/druid/msq/sql/resources/SqlTaskResource.java  | 2 +-
 .../druid/testsEx/catalog/ITCatalogIngestAndQueryTest.java   | 2 +-
 .../druid/testsEx/catalog/ITCatalogIngestErrorTest.java      | 2 +-
 .../druid/testsEx/msq/ITKeyStatisticsSketchMergeMode.java    | 2 +-
 .../testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java   | 2 +-
 .../java/org/apache/druid/testsEx/msq/MultiStageQuery.java   | 2 +-
 .../org/apache/druid/testing/utils/MsqTestQueryHelper.java   | 2 +-
 .../main/java/org/apache/druid/sql/http}/SqlTaskStatus.java  | 5 ++---
 .../java/org/apache/druid/sql/http}/SqlTaskStatusTest.java   | 2 +-
 10 files changed, 11 insertions(+), 12 deletions(-)
 rename {extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql => sql/src/main/java/org/apache/druid/sql/http}/SqlTaskStatus.java (94%)
 rename {extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql => sql/src/test/java/org/apache/druid/sql/http}/SqlTaskStatusTest.java (98%)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/TaskStartTimeoutFault.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/TaskStartTimeoutFault.java
index 37d464a324e7..1eba2e8dc779 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/TaskStartTimeoutFault.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/error/TaskStartTimeoutFault.java
@@ -45,7 +45,7 @@ public TaskStartTimeoutFault(
   {
     super(
         CODE,
-        "Unable to launch [%d] workers out of the total [%d] worker tasks within [%,d] seconds of the last successful worker launch."
+        "Unable to launch [%d] workers out of the total [%d] worker tasks within [%,d] seconds of the last successful worker launch. "
         + "There might be insufficient available slots to start all worker tasks simultaneously. "
         + "Try lowering '%s' in your query context to a number that fits within your available task capacity, "
         + "or try increasing capacity.",
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlTaskResource.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlTaskResource.java
index 7a51bc8d26a4..a07ef2c403fb 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlTaskResource.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/resources/SqlTaskResource.java
@@ -33,7 +33,6 @@
 import org.apache.druid.java.util.common.logger.Logger;
 import org.apache.druid.msq.guice.MultiStageQuery;
 import org.apache.druid.msq.sql.MSQTaskSqlEngine;
-import org.apache.druid.msq.sql.SqlTaskStatus;
 import org.apache.druid.query.QueryException;
 import org.apache.druid.server.QueryResponse;
 import org.apache.druid.server.initialization.ServerConfig;
@@ -48,6 +47,7 @@
 import org.apache.druid.sql.http.ResultFormat;
 import org.apache.druid.sql.http.SqlQuery;
 import org.apache.druid.sql.http.SqlResource;
+import org.apache.druid.sql.http.SqlTaskStatus;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.Consumes;
diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/catalog/ITCatalogIngestAndQueryTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/catalog/ITCatalogIngestAndQueryTest.java
index 29dee7329cb0..1a756910bf24 100644
--- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/catalog/ITCatalogIngestAndQueryTest.java
+++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/catalog/ITCatalogIngestAndQueryTest.java
@@ -29,9 +29,9 @@
 import org.apache.druid.catalog.model.table.TableBuilder;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.msq.sql.SqlTaskStatus;
 import org.apache.druid.query.QueryContexts;
 import org.apache.druid.sql.http.SqlQuery;
+import org.apache.druid.sql.http.SqlTaskStatus;
 import org.apache.druid.testing.utils.DataLoaderHelper;
 import org.apache.druid.testing.utils.MsqTestQueryHelper;
 import org.apache.druid.testsEx.cluster.CatalogClient;
diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/catalog/ITCatalogIngestErrorTest.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/catalog/ITCatalogIngestErrorTest.java
index 87e35e2b6465..e075462d0bb6 100644
--- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/catalog/ITCatalogIngestErrorTest.java
+++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/catalog/ITCatalogIngestErrorTest.java
@@ -27,8 +27,8 @@
 import org.apache.druid.catalog.model.table.DatasourceDefn;
 import org.apache.druid.catalog.model.table.TableBuilder;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.msq.sql.SqlTaskStatus;
 import org.apache.druid.sql.http.SqlQuery;
+import org.apache.druid.sql.http.SqlTaskStatus;
 import org.apache.druid.testing.utils.MsqTestQueryHelper;
 import org.apache.druid.testsEx.categories.Catalog;
 import org.apache.druid.testsEx.cluster.CatalogClient;
diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITKeyStatisticsSketchMergeMode.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITKeyStatisticsSketchMergeMode.java
index ad0b764e70fe..b0dee9e713eb 100644
--- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITKeyStatisticsSketchMergeMode.java
+++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITKeyStatisticsSketchMergeMode.java
@@ -23,9 +23,9 @@
 import com.google.inject.Inject;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.msq.exec.ClusterStatisticsMergeMode;
-import org.apache.druid.msq.sql.SqlTaskStatus;
 import org.apache.druid.msq.util.MultiStageQueryContext;
 import org.apache.druid.sql.http.SqlQuery;
+import org.apache.druid.sql.http.SqlTaskStatus;
 import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
 import org.apache.druid.testing.utils.DataLoaderHelper;
 import org.apache.druid.testing.utils.MsqTestQueryHelper;
diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java
index 37d99b7d9b93..f45ce3ad0e96 100644
--- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java
+++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/ITMultiStageQueryWorkerFaultTolerance.java
@@ -25,8 +25,8 @@
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.msq.sql.SqlTaskStatus;
 import org.apache.druid.msq.util.MultiStageQueryContext;
+import org.apache.druid.sql.http.SqlTaskStatus;
 import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
 import org.apache.druid.testing.utils.DataLoaderHelper;
 import org.apache.druid.testing.utils.ITRetryUtil;
diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/MultiStageQuery.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/MultiStageQuery.java
index bda1c2434536..3cdc57c37bfa 100644
--- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/MultiStageQuery.java
+++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/msq/MultiStageQuery.java
@@ -29,7 +29,7 @@
 import org.apache.druid.msq.indexing.report.MSQResultsReport;
 import org.apache.druid.msq.indexing.report.MSQTaskReport;
 import org.apache.druid.msq.indexing.report.MSQTaskReportPayload;
-import org.apache.druid.msq.sql.SqlTaskStatus;
+import org.apache.druid.sql.http.SqlTaskStatus;
 import org.apache.druid.storage.local.LocalFileExportStorageProvider;
 import org.apache.druid.testing.clients.CoordinatorResourceTestClient;
 import org.apache.druid.testing.utils.DataLoaderHelper;
diff --git a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java
index 4efd5c7d42d8..2a4d04f42dbd 100644
--- a/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java
+++ b/integration-tests/src/main/java/org/apache/druid/testing/utils/MsqTestQueryHelper.java
@@ -37,8 +37,8 @@
 import org.apache.druid.msq.indexing.report.MSQResultsReport;
 import org.apache.druid.msq.indexing.report.MSQTaskReport;
 import org.apache.druid.msq.indexing.report.MSQTaskReportPayload;
-import org.apache.druid.msq.sql.SqlTaskStatus;
 import org.apache.druid.sql.http.SqlQuery;
+import org.apache.druid.sql.http.SqlTaskStatus;
 import org.apache.druid.testing.IntegrationTestingConfig;
 import org.apache.druid.testing.clients.OverlordResourceTestClient;
 import org.apache.druid.testing.clients.SqlResourceTestClient;
diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/SqlTaskStatus.java b/sql/src/main/java/org/apache/druid/sql/http/SqlTaskStatus.java
similarity index 94%
rename from extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/SqlTaskStatus.java
rename to sql/src/main/java/org/apache/druid/sql/http/SqlTaskStatus.java
index 9c92a2589f2d..765891ec20a1 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/sql/SqlTaskStatus.java
+++ b/sql/src/main/java/org/apache/druid/sql/http/SqlTaskStatus.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.druid.msq.sql;
+package org.apache.druid.sql.http;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonInclude;
@@ -25,13 +25,12 @@
 import com.google.common.base.Preconditions;
 import org.apache.druid.error.ErrorResponse;
 import org.apache.druid.indexer.TaskState;
-import org.apache.druid.msq.sql.resources.SqlTaskResource;
 
 import javax.annotation.Nullable;
 import java.util.Objects;
 
 /**
- * Response object for {@link SqlTaskResource#doPost}.
+ * Response object for org.apache.druid.msq.sql.resources.SqlTaskResource#doPost.
  */
 public class SqlTaskStatus
 {
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/SqlTaskStatusTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlTaskStatusTest.java
similarity index 98%
rename from extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/SqlTaskStatusTest.java
rename to sql/src/test/java/org/apache/druid/sql/http/SqlTaskStatusTest.java
index 301f91ce8d3d..c73e8d3cddbd 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/sql/SqlTaskStatusTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/http/SqlTaskStatusTest.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.druid.msq.sql;
+package org.apache.druid.sql.http;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import nl.jqno.equalsverifier.EqualsVerifier;

From 5da99499926395c7c5fe2cc23b56c8e358c04fd9 Mon Sep 17 00:00:00 2001
From: Vishesh Garg <gargvishesh@gmail.com>
Date: Sat, 19 Oct 2024 13:33:33 +0530
Subject: [PATCH 02/12] Fail MSQ compaction if multi-valued partition
 dimensions are found  (#17344)

MSQ currently supports only single-valued string dimensions as partition keys.
This patch adds a check to ensure that partition keys are single-valued in case
this info is available by virtue of segment download for schema inference.

During compaction, if MSQ finds multi-valued dimensions (MVDs) declared as part
of `range` partitionsSpec, it switches partitioning type to dynamic, ending up in
repeated compactions of the same interval. To avoid this scenario, the segment
download logic is also updated to always download segments if info on multi-valued
dimensions is required.
---
 .../msq/indexing/MSQCompactionRunner.java     |  43 ++++++-
 .../msq/indexing/MSQCompactionRunnerTest.java |  58 +++++++--
 .../indexing/common/task/CompactionTask.java  |  66 ++++++++--
 .../common/task/CompactionTaskTest.java       | 117 ++++++++++++++++++
 .../indexing/ClientCompactionRunnerInfo.java  |   9 +-
 .../ClientCompactionRunnerInfoTest.java       |  29 ++++-
 6 files changed, 296 insertions(+), 26 deletions(-)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
index e20188d58294..d05ab12ea3fe 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQCompactionRunner.java
@@ -29,6 +29,7 @@
 import com.google.inject.Injector;
 import org.apache.druid.client.indexing.ClientCompactionRunnerInfo;
 import org.apache.druid.data.input.impl.DimensionSchema;
+import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.indexer.TaskStatus;
 import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
 import org.apache.druid.indexer.partitions.PartitionsSpec;
@@ -84,6 +85,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -130,7 +132,7 @@ public MSQCompactionRunner(
    * The following configs aren't supported:
    * <ul>
    * <li>partitionsSpec of type HashedParititionsSpec.</li>
-   * <li>'range' partitionsSpec with non-string partition dimensions.</li>
+   * <li>'range' partitionsSpec with multi-valued or non-string partition dimensions.</li>
    * <li>maxTotalRows in DynamicPartitionsSpec.</li>
    * <li>Rollup without metricsSpec being specified or vice-versa.</li>
    * <li>Any aggregatorFactory {@code A} s.t. {@code A != A.combiningFactory()}.</li>
@@ -153,13 +155,24 @@ public CompactionConfigValidationResult validateCompactionTask(
       );
     }
     List<CompactionConfigValidationResult> validationResults = new ArrayList<>();
+    DataSchema dataSchema = Iterables.getOnlyElement(intervalToDataSchemaMap.values());
     if (compactionTask.getTuningConfig() != null) {
       validationResults.add(
           ClientCompactionRunnerInfo.validatePartitionsSpecForMSQ(
               compactionTask.getTuningConfig().getPartitionsSpec(),
-              Iterables.getOnlyElement(intervalToDataSchemaMap.values()).getDimensionsSpec().getDimensions()
+              dataSchema.getDimensionsSpec().getDimensions()
           )
       );
+      validationResults.add(
+          validatePartitionDimensionsAreNotMultiValued(
+              compactionTask.getTuningConfig().getPartitionsSpec(),
+              dataSchema.getDimensionsSpec(),
+              dataSchema instanceof CombinedDataSchema
+              ? ((CombinedDataSchema) dataSchema).getMultiValuedDimensions()
+              : null
+          )
+      );
+
     }
     if (compactionTask.getGranularitySpec() != null) {
       validationResults.add(ClientCompactionRunnerInfo.validateRollupForMSQ(
@@ -175,6 +188,32 @@ public CompactionConfigValidationResult validateCompactionTask(
                             .orElse(CompactionConfigValidationResult.success());
   }
 
+  private CompactionConfigValidationResult validatePartitionDimensionsAreNotMultiValued(
+      PartitionsSpec partitionsSpec,
+      DimensionsSpec dimensionsSpec,
+      Set<String> multiValuedDimensions
+  )
+  {
+    List<String> dimensionSchemas = dimensionsSpec.getDimensionNames();
+    if (partitionsSpec instanceof DimensionRangePartitionsSpec
+        && dimensionSchemas != null
+        && multiValuedDimensions != null
+        && !multiValuedDimensions.isEmpty()) {
+      Optional<String> multiValuedDimension = ((DimensionRangePartitionsSpec) partitionsSpec)
+          .getPartitionDimensions()
+          .stream()
+          .filter(multiValuedDimensions::contains)
+          .findAny();
+      if (multiValuedDimension.isPresent()) {
+        return CompactionConfigValidationResult.failure(
+            "MSQ: Multi-valued string partition dimension[%s] not supported with 'range' partition spec",
+            multiValuedDimension.get()
+        );
+      }
+    }
+    return CompactionConfigValidationResult.success();
+  }
+
   @Override
   public CurrentSubTaskHolder getCurrentSubTaskHolder()
   {
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
index 0b5395d727fe..0a54f8550a93 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/indexing/MSQCompactionRunnerTest.java
@@ -23,6 +23,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import org.apache.druid.client.indexing.ClientCompactionTaskGranularitySpec;
 import org.apache.druid.client.indexing.ClientCompactionTaskTransformSpec;
@@ -109,11 +110,15 @@ public class MSQCompactionRunnerTest
   );
   private static final Map<Interval, DataSchema> INTERVAL_DATASCHEMAS = ImmutableMap.of(
       COMPACTION_INTERVAL,
-      new DataSchema.Builder()
-          .withDataSource(DATA_SOURCE)
-          .withTimestamp(new TimestampSpec(TIMESTAMP_COLUMN, null, null))
-          .withDimensions(new DimensionsSpec(DIMENSIONS))
-          .build()
+      new CombinedDataSchema(
+          DATA_SOURCE,
+          new TimestampSpec(TIMESTAMP_COLUMN, null, null),
+          new DimensionsSpec(DIMENSIONS),
+          null,
+          null,
+          null,
+          ImmutableSet.of(MV_STRING_DIMENSION.getName())
+      )
   );
   private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
   private static final AggregatorFactory AGG1 = new CountAggregatorFactory("agg_0");
@@ -139,6 +144,7 @@ public void testMultipleDisjointCompactionIntervalsAreInvalid()
         null,
         Collections.emptyMap(),
         null,
+        null,
         null
     );
     CompactionConfigValidationResult validationResult = MSQ_COMPACTION_RUNNER.validateCompactionTask(
@@ -160,6 +166,7 @@ public void testHashedPartitionsSpecIsInvalid()
         null,
         Collections.emptyMap(),
         null,
+        null,
         null
     );
     Assert.assertFalse(MSQ_COMPACTION_RUNNER.validateCompactionTask(compactionTask, INTERVAL_DATASCHEMAS).isValid());
@@ -173,6 +180,7 @@ public void testStringDimensionInRangePartitionsSpecIsValid()
         null,
         Collections.emptyMap(),
         null,
+        null,
         null
     );
     Assert.assertTrue(MSQ_COMPACTION_RUNNER.validateCompactionTask(compactionTask, INTERVAL_DATASCHEMAS).isValid());
@@ -187,6 +195,7 @@ public void testLongDimensionInRangePartitionsSpecIsInvalid()
         null,
         Collections.emptyMap(),
         null,
+        null,
         null
     );
 
@@ -200,6 +209,29 @@ public void testLongDimensionInRangePartitionsSpecIsInvalid()
     );
   }
 
+  @Test
+  public void testMultiValuedDimensionInRangePartitionsSpecIsInvalid()
+  {
+    List<String> mvStringPartitionDimension = Collections.singletonList(MV_STRING_DIMENSION.getName());
+    CompactionTask compactionTask = createCompactionTask(
+        new DimensionRangePartitionsSpec(TARGET_ROWS_PER_SEGMENT, null, mvStringPartitionDimension, false),
+        null,
+        Collections.emptyMap(),
+        null,
+        null,
+        null
+    );
+
+    CompactionConfigValidationResult validationResult = MSQ_COMPACTION_RUNNER.validateCompactionTask(compactionTask,
+                                                                                                     INTERVAL_DATASCHEMAS
+    );
+    Assert.assertFalse(validationResult.isValid());
+    Assert.assertEquals(
+        "MSQ: Multi-valued string partition dimension[mv_string_dim] not supported with 'range' partition spec",
+        validationResult.getReason()
+    );
+  }
+
   @Test
   public void testMaxTotalRowsIsInvalid()
   {
@@ -208,6 +240,7 @@ public void testMaxTotalRowsIsInvalid()
         null,
         Collections.emptyMap(),
         null,
+        null,
         null
     );
     Assert.assertFalse(MSQ_COMPACTION_RUNNER.validateCompactionTask(compactionTask, INTERVAL_DATASCHEMAS).isValid());
@@ -221,6 +254,7 @@ public void testDynamicPartitionsSpecIsValid()
         null,
         Collections.emptyMap(),
         null,
+        null,
         null
     );
     Assert.assertTrue(MSQ_COMPACTION_RUNNER.validateCompactionTask(compactionTask, INTERVAL_DATASCHEMAS).isValid());
@@ -234,6 +268,7 @@ public void testQueryGranularityAllIsValid()
         null,
         Collections.emptyMap(),
         new ClientCompactionTaskGranularitySpec(null, Granularities.ALL, null),
+        null,
         null
     );
     Assert.assertTrue(MSQ_COMPACTION_RUNNER.validateCompactionTask(compactionTask, INTERVAL_DATASCHEMAS).isValid());
@@ -247,26 +282,28 @@ public void testRollupFalseWithMetricsSpecIsInValid()
         null,
         Collections.emptyMap(),
         new ClientCompactionTaskGranularitySpec(null, null, false),
+        null,
         AGGREGATORS.toArray(new AggregatorFactory[0])
     );
     Assert.assertFalse(MSQ_COMPACTION_RUNNER.validateCompactionTask(compactionTask, INTERVAL_DATASCHEMAS).isValid());
   }
 
   @Test
-  public void testRollupTrueWithoutMetricsSpecIsInValid()
+  public void testRollupTrueWithoutMetricsSpecIsInvalid()
   {
     CompactionTask compactionTask = createCompactionTask(
         new DynamicPartitionsSpec(3, null),
         null,
         Collections.emptyMap(),
         new ClientCompactionTaskGranularitySpec(null, null, true),
+        null,
         null
     );
     Assert.assertFalse(MSQ_COMPACTION_RUNNER.validateCompactionTask(compactionTask, INTERVAL_DATASCHEMAS).isValid());
   }
 
   @Test
-  public void testMSQEngineWithUnsupportedMetricsSpecIsInValid()
+  public void testMSQEngineWithUnsupportedMetricsSpecIsInvalid()
   {
     // Aggregators having different input and ouput column names are unsupported.
     final String inputColName = "added";
@@ -276,6 +313,7 @@ public void testMSQEngineWithUnsupportedMetricsSpecIsInValid()
         null,
         Collections.emptyMap(),
         new ClientCompactionTaskGranularitySpec(null, null, true),
+        null,
         new AggregatorFactory[]{new LongSumAggregatorFactory(outputColName, inputColName)}
     );
     CompactionConfigValidationResult validationResult = MSQ_COMPACTION_RUNNER.validateCompactionTask(
@@ -292,7 +330,7 @@ public void testMSQEngineWithUnsupportedMetricsSpecIsInValid()
   @Test
   public void testRunCompactionTasksWithEmptyTaskListFails() throws Exception
   {
-    CompactionTask compactionTask = createCompactionTask(null, null, Collections.emptyMap(), null, null);
+    CompactionTask compactionTask = createCompactionTask(null, null, Collections.emptyMap(), null, null, null);
     TaskStatus taskStatus = MSQ_COMPACTION_RUNNER.runCompactionTasks(compactionTask, Collections.emptyMap(), null);
     Assert.assertTrue(taskStatus.isFailure());
   }
@@ -307,6 +345,7 @@ public void testMSQControllerTaskSpecWithScanIsValid() throws JsonProcessingExce
         dimFilter,
         Collections.emptyMap(),
         null,
+        null,
         null
     );
 
@@ -384,6 +423,7 @@ public void testMSQControllerTaskSpecWithAggregatorsIsValid() throws JsonProcess
         dimFilter,
         Collections.emptyMap(),
         null,
+        null,
         null
     );
 
@@ -481,6 +521,7 @@ private CompactionTask createCompactionTask(
       @Nullable DimFilter dimFilter,
       Map<String, Object> contextParams,
       @Nullable ClientCompactionTaskGranularitySpec granularitySpec,
+      @Nullable List<DimensionSchema> dimensionSchemas,
       @Nullable AggregatorFactory[] metricsSpec
   )
   {
@@ -504,6 +545,7 @@ private CompactionTask createCompactionTask(
         ))
         .transformSpec(transformSpec)
         .granularitySpec(granularitySpec)
+        .dimensionsSpec(new DimensionsSpec(dimensionSchemas))
         .metricsSpec(metricsSpec)
         .compactionRunner(MSQ_COMPACTION_RUNNER)
         .context(context);
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
index 4594fc1e9b25..dc2eb75e3cc8 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/CompactionTask.java
@@ -48,6 +48,7 @@
 import org.apache.druid.indexer.Checks;
 import org.apache.druid.indexer.Property;
 import org.apache.druid.indexer.TaskStatus;
+import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
 import org.apache.druid.indexer.partitions.PartitionsSpec;
 import org.apache.druid.indexing.common.LockGranularity;
 import org.apache.druid.indexing.common.SegmentCacheManagerFactory;
@@ -80,6 +81,7 @@
 import org.apache.druid.segment.QueryableIndex;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ColumnHolder;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.incremental.AppendableIndexSpec;
 import org.apache.druid.segment.indexing.CombinedDataSchema;
 import org.apache.druid.segment.indexing.DataSchema;
@@ -452,6 +454,50 @@ public boolean isPerfectRollup()
     return tuningConfig != null && tuningConfig.isForceGuaranteedRollup();
   }
 
+  /**
+   * Checks if multi-valued string dimensions need to be analyzed by downloading the segments.
+   * This method returns true only for MSQ engine when either of the following holds true:
+   * <ul>
+   * <li> Range partitioning is done on a string dimension or an unknown dimension
+   * (since MSQ does not support partitioning on a multi-valued string dimension) </li>
+   * <li> Rollup is done on a string dimension or an unknown dimension
+   * (since MSQ requires multi-valued string dimensions to be converted to arrays for rollup) </li>
+   * </ul>
+   * @return false for native engine, true for MSQ engine only when partitioning or rollup is done on a string
+   * or unknown dimension.
+   */
+  boolean identifyMultiValuedDimensions()
+  {
+    if (compactionRunner instanceof NativeCompactionRunner) {
+      return false;
+    }
+    // Rollup can be true even when granularitySpec is not known since rollup is then decided based on segment analysis
+    final boolean isPossiblyRollup = granularitySpec == null || !Boolean.FALSE.equals(granularitySpec.isRollup());
+    boolean isRangePartitioned = tuningConfig != null
+                                 && tuningConfig.getPartitionsSpec() instanceof DimensionRangePartitionsSpec;
+
+    if (dimensionsSpec == null || dimensionsSpec.getDimensions().isEmpty()) {
+      return isPossiblyRollup || isRangePartitioned;
+    } else {
+      boolean isRollupOnStringDimension = isPossiblyRollup &&
+                                          dimensionsSpec.getDimensions()
+                                                        .stream()
+                                                        .anyMatch(dim -> ColumnType.STRING.equals(dim.getColumnType()));
+
+      boolean isPartitionedOnStringDimension =
+          isRangePartitioned &&
+          dimensionsSpec.getDimensions()
+                        .stream()
+                        .anyMatch(
+                            dim -> ColumnType.STRING.equals(dim.getColumnType())
+                                   && ((DimensionRangePartitionsSpec) tuningConfig.getPartitionsSpec())
+                                       .getPartitionDimensions()
+                                       .contains(dim.getName())
+                        );
+      return isRollupOnStringDimension || isPartitionedOnStringDimension;
+    }
+  }
+
   @Override
   public TaskStatus runTask(TaskToolbox toolbox) throws Exception
   {
@@ -466,7 +512,7 @@ public TaskStatus runTask(TaskToolbox toolbox) throws Exception
         metricsSpec,
         granularitySpec,
         getMetricBuilder(),
-        !(compactionRunner instanceof NativeCompactionRunner)
+        this.identifyMultiValuedDimensions()
     );
 
     registerResourceCloserOnAbnormalExit(compactionRunner.getCurrentSubTaskHolder());
@@ -794,23 +840,25 @@ static class ExistingSegmentAnalyzer
       this.needMultiValuedDimensions = needMultiValuedDimensions;
     }
 
-    private boolean shouldFetchSegments()
+    /**
+     * Segments are downloaded even when just needMultiValuedDimensions=true since MSQ switches to dynamic partitioning
+     * on finding any 'range' partition dimension to be multivalued at runtime, which ends up in a mismatch between
+     * the compaction config and the actual segments (lastCompactionState), leading to repeated compactions.
+     */
+    private boolean shouldDownloadSegments()
     {
-      // Don't fetch segments for just needMultiValueDimensions
-      return needRollup || needQueryGranularity || needDimensionsSpec || needMetricsSpec;
+
+      return needRollup || needQueryGranularity || needDimensionsSpec || needMetricsSpec || needMultiValuedDimensions;
     }
 
     public void fetchAndProcessIfNeeded()
     {
-      if (!shouldFetchSegments()) {
+      if (!shouldDownloadSegments()) {
         // Nothing to do; short-circuit and don't fetch segments.
         return;
       }
 
-      if (needMultiValuedDimensions) {
-        multiValuedDimensions = new HashSet<>();
-      }
-
+      multiValuedDimensions = new HashSet<>();
       final List<Pair<DataSegment, Supplier<ResourceHolder<QueryableIndex>>>> segments = sortSegmentsListNewestFirst();
 
       for (Pair<DataSegment, Supplier<ResourceHolder<QueryableIndex>>> segmentPair : segments) {
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
index 8908d0d50c74..f2a795fec043 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java
@@ -55,6 +55,7 @@
 import org.apache.druid.guice.GuiceInjectors;
 import org.apache.druid.guice.IndexingServiceTuningConfigModule;
 import org.apache.druid.indexer.TaskStatus;
+import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
 import org.apache.druid.indexer.partitions.HashedPartitionsSpec;
 import org.apache.druid.indexing.common.LockGranularity;
 import org.apache.druid.indexing.common.SegmentCacheManagerFactory;
@@ -129,6 +130,7 @@
 import org.apache.druid.segment.realtime.appenderator.AppenderatorsManager;
 import org.apache.druid.segment.selector.settable.SettableColumnValueSelector;
 import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
+import org.apache.druid.server.coordinator.CompactionConfigValidationResult;
 import org.apache.druid.server.lookup.cache.LookupLoadingSpec;
 import org.apache.druid.server.security.AuthTestUtils;
 import org.apache.druid.server.security.AuthorizerMapper;
@@ -1533,6 +1535,89 @@ public void testMultiValuedDimensionsProcessing()
     }
   }
 
+  @Test
+  public void testMSQRollupWithNoDimensionsSpecNeedsMVDInfo()
+  {
+    final Builder builder = new Builder(
+        DATA_SOURCE,
+        segmentCacheManagerFactory
+    );
+    builder.inputSpec(new CompactionIntervalSpec(COMPACTION_INTERVAL, SegmentUtils.hashIds(SEGMENTS)));
+    builder.compactionRunner(new TestMSQCompactionRunner());
+    final CompactionTask compactionTask = builder.build();
+    // granularitySpec=null should assume a possible rollup
+    Assert.assertTrue(compactionTask.identifyMultiValuedDimensions());
+  }
+
+  @Test
+  public void testMSQRangePartitionWithNoDimensionsSpecNeedsMVDInfo()
+  {
+    final Builder builder = new Builder(
+        DATA_SOURCE,
+        segmentCacheManagerFactory
+    );
+    builder.inputSpec(new CompactionIntervalSpec(COMPACTION_INTERVAL, SegmentUtils.hashIds(SEGMENTS)));
+    builder.compactionRunner(new TestMSQCompactionRunner());
+    builder.tuningConfig(TuningConfigBuilder.forCompactionTask()
+                                            .withForceGuaranteedRollup(true)
+                                            .withPartitionsSpec(
+                                                new DimensionRangePartitionsSpec(
+                                                    3,
+                                                    null,
+                                                    ImmutableList.of(
+                                                        "string_dim_1"),
+                                                    false
+                                                ))
+                                            .build());
+    final CompactionTask compactionTask = builder.build();
+    Assert.assertTrue(compactionTask.identifyMultiValuedDimensions());
+  }
+
+  @Test
+  public void testMSQRollupOnStringNeedsMVDInfo()
+  {
+    final Builder builder = new Builder(
+        DATA_SOURCE,
+        segmentCacheManagerFactory
+    );
+    builder.inputSpec(new CompactionIntervalSpec(COMPACTION_INTERVAL, SegmentUtils.hashIds(SEGMENTS)));
+    builder.compactionRunner(new TestMSQCompactionRunner());
+    builder.granularitySpec(new ClientCompactionTaskGranularitySpec(null, null, true));
+
+    DimensionSchema stringDim = new StringDimensionSchema("string_dim_1", null, null);
+    builder.dimensionsSpec(new DimensionsSpec(ImmutableList.of(stringDim)));
+    final CompactionTask compactionTask = builder.build();
+    // A string dimension with rollup=true should need MVD info
+    Assert.assertTrue(compactionTask.identifyMultiValuedDimensions());
+  }
+
+  @Test
+  public void testMSQRangePartitionOnStringNeedsMVDInfo()
+  {
+    final Builder builder = new Builder(
+        DATA_SOURCE,
+        segmentCacheManagerFactory
+    );
+    builder.inputSpec(new CompactionIntervalSpec(COMPACTION_INTERVAL, SegmentUtils.hashIds(SEGMENTS)));
+    builder.compactionRunner(new TestMSQCompactionRunner());
+
+    DimensionSchema stringDim = new StringDimensionSchema("string_dim_1", null, null);
+    builder.tuningConfig(TuningConfigBuilder.forCompactionTask()
+                                            .withForceGuaranteedRollup(true)
+                                            .withPartitionsSpec(
+                                                new DimensionRangePartitionsSpec(
+                                                    3,
+                                                    null,
+                                                    ImmutableList.of(
+                                                        stringDim.getName()),
+                                                    false
+                                                ))
+                                            .build());
+    builder.dimensionsSpec(new DimensionsSpec(ImmutableList.of(stringDim)));
+    CompactionTask compactionTask = builder.build();
+    Assert.assertTrue(compactionTask.identifyMultiValuedDimensions());
+  }
+
   @Test
   public void testChooseFinestGranularityWithNulls()
   {
@@ -2015,6 +2100,38 @@ public SettableColumnValueSelector<?> makeNewSettableColumnValueSelector()
 
   }
 
+  /**
+   * A class to mimic validations with MSQCompactionRunner behaviour, since the actual class resides in the MSQ extn.
+   * Since validations just depend on the type of runner, all overrideen functions just return null.
+   */
+  private static class TestMSQCompactionRunner implements CompactionRunner
+  {
+    @Override
+    public TaskStatus runCompactionTasks(
+        CompactionTask compactionTask,
+        Map<Interval, DataSchema> intervalDataSchemaMap,
+        TaskToolbox taskToolbox
+    )
+    {
+      return null;
+    }
+
+    @Override
+    public CurrentSubTaskHolder getCurrentSubTaskHolder()
+    {
+      return null;
+    }
+
+    @Override
+    public CompactionConfigValidationResult validateCompactionTask(
+        CompactionTask compactionTask,
+        Map<Interval, DataSchema> intervalToDataSchemaMap
+    )
+    {
+      return null;
+    }
+  }
+
   /**
    * The compaction task spec in 0.16.0 except for the tuningConfig.
    * The original spec accepts only {@link IndexTuningConfig}, but this class acceps any type of tuningConfig for
diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionRunnerInfo.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionRunnerInfo.java
index f6a009afe1ce..532c71977189 100644
--- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionRunnerInfo.java
+++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactionRunnerInfo.java
@@ -107,7 +107,7 @@ public static CompactionConfigValidationResult validateCompactionConfig(
    * Checks if the provided compaction config is supported by MSQ. The following configs aren't supported:
    * <ul>
    * <li>partitionsSpec of type HashedParititionsSpec.</li>
-   * <li>'range' partitionsSpec with non-string partition dimensions.</li>
+   * <li>'range' partitionsSpec with multi-valued or non-string partition dimensions.</li>
    * <li>maxTotalRows in DynamicPartitionsSpec.</li>
    * <li>Rollup without metricsSpec being specified or vice-versa.</li>
    * <li>Any aggregatorFactory {@code A} s.t. {@code A != A.combiningFactory()}.</li>
@@ -138,13 +138,18 @@ private static CompactionConfigValidationResult compactionConfigSupportedByMSQEn
 
   /**
    * Validate that partitionSpec is either 'dynamic` or 'range'. If 'dynamic', ensure 'maxTotalRows' is null. If range
-   * ensure all partition columns are of string type.
+   * ensure all partition columns are of type string.
    */
   public static CompactionConfigValidationResult validatePartitionsSpecForMSQ(
       @Nullable PartitionsSpec partitionsSpec,
       @Nullable List<DimensionSchema> dimensionSchemas
   )
   {
+    if (partitionsSpec == null) {
+      return CompactionConfigValidationResult.failure(
+          "MSQ: tuningConfig.partitionsSpec must be specified"
+      );
+    }
     if (!(partitionsSpec instanceof DimensionRangePartitionsSpec
           || partitionsSpec instanceof DynamicPartitionsSpec)) {
       return CompactionConfigValidationResult.failure(
diff --git a/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionRunnerInfoTest.java b/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionRunnerInfoTest.java
index b1f065422805..449915a78462 100644
--- a/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionRunnerInfoTest.java
+++ b/server/src/test/java/org/apache/druid/client/indexing/ClientCompactionRunnerInfoTest.java
@@ -95,6 +95,27 @@ public void testMSQEngineWithMaxTotalRowsIsInvalid()
     );
   }
 
+  @Test
+  public void testMSQEngineWithNullPartitionsSpecIsInvalid()
+  {
+    DataSourceCompactionConfig compactionConfig = createMSQCompactionConfig(
+        null,
+        Collections.emptyMap(),
+        null,
+        null,
+        null
+    );
+    CompactionConfigValidationResult validationResult = ClientCompactionRunnerInfo.validateCompactionConfig(
+        compactionConfig,
+        CompactionEngine.NATIVE
+    );
+    Assert.assertFalse(validationResult.isValid());
+    Assert.assertEquals(
+        "MSQ: tuningConfig.partitionsSpec must be specified",
+        validationResult.getReason()
+    );
+  }
+
   @Test
   public void testMSQEngineWithDynamicPartitionsSpecIsValid()
   {
@@ -124,7 +145,7 @@ public void testMSQEngineWithStringDimensionsInRangePartitionsSpecIsValid()
   }
 
   @Test
-  public void testMSQEngineWithLongDimensionsInRangePartitionsSpecIsValid()
+  public void testMSQEngineWithLongDimensionsInRangePartitionsSpecIsInvalid()
   {
     DataSourceCompactionConfig compactionConfig = createMSQCompactionConfig(
         new DimensionRangePartitionsSpec(100, null, ImmutableList.of("partitionDim"), false),
@@ -253,7 +274,7 @@ private static DataSourceCompactionConfig createMSQCompactionConfig(
       List<DimensionSchema> dimensions
   )
   {
-    final DataSourceCompactionConfig config = new DataSourceCompactionConfig(
+    return new DataSourceCompactionConfig(
         "dataSource",
         null,
         500L,
@@ -268,12 +289,11 @@ private static DataSourceCompactionConfig createMSQCompactionConfig(
         CompactionEngine.MSQ,
         context
     );
-    return config;
   }
 
   private static UserCompactionTaskQueryTuningConfig createTuningConfig(PartitionsSpec partitionsSpec)
   {
-    final UserCompactionTaskQueryTuningConfig tuningConfig = new UserCompactionTaskQueryTuningConfig(
+    return new UserCompactionTaskQueryTuningConfig(
         40000,
         null,
         2000L,
@@ -302,6 +322,5 @@ private static UserCompactionTaskQueryTuningConfig createTuningConfig(Partitions
         100,
         2
     );
-    return tuningConfig;
   }
 }

From 187e21afaea8495d94ed780254beef9b12536757 Mon Sep 17 00:00:00 2001
From: Abhishek Radhakrishnan <abhishek.rb19@gmail.com>
Date: Mon, 21 Oct 2024 11:05:53 -0700
Subject: [PATCH 03/12] Add `BrokerClient` implementation (#17382)

This patch is extracted from PR 17353.

Changes:

- Added BrokerClient and BrokerClientImpl to the sql package that leverages the ServiceClient functionality; similar to OverlordClient and CoordinatorClient implementations in the server module.
- For now, only two broker API stubs are added: submitSqlTask() and fetchExplainPlan().
- Added a new POJO class ExplainPlan that encapsulates explain plan info.
- Deprecated org.apache.druid.discovery.BrokerClient in favor of the new BrokerClient in this patch.
- Clean up ExplainAttributesTest a bit and added serde verification.
---
 .../apache/druid/discovery/BrokerClient.java  |   4 +
 .../druid/rpc/guice/ServiceClientModule.java  |  15 +-
 .../rpc/guice/ServiceClientModuleTest.java    |  99 ++++++++++++
 .../calcite/planner/ExplainAttributes.java    |  26 +++
 .../org/apache/druid/sql/client/Broker.java   |  34 ++++
 .../apache/druid/sql/client/BrokerClient.java |  51 ++++++
 .../druid/sql/client/BrokerClientImpl.java    |  84 ++++++++++
 .../druid/sql/guice/BrokerServiceModule.java  |  91 +++++++++++
 .../apache/druid/sql/http/ExplainPlan.java    | 117 ++++++++++++++
 .../planner/ExplainAttributesTest.java        |  81 ++++++----
 .../sql/client/BrokerClientImplTest.java      | 148 ++++++++++++++++++
 .../sql/guice/BrokerServiceModuleTest.java    | 114 ++++++++++++++
 .../druid/sql/http/ExplainPlanTest.java       | 122 +++++++++++++++
 13 files changed, 953 insertions(+), 33 deletions(-)
 create mode 100644 server/src/test/java/org/apache/druid/rpc/guice/ServiceClientModuleTest.java
 create mode 100644 sql/src/main/java/org/apache/druid/sql/client/Broker.java
 create mode 100644 sql/src/main/java/org/apache/druid/sql/client/BrokerClient.java
 create mode 100644 sql/src/main/java/org/apache/druid/sql/client/BrokerClientImpl.java
 create mode 100644 sql/src/main/java/org/apache/druid/sql/guice/BrokerServiceModule.java
 create mode 100644 sql/src/main/java/org/apache/druid/sql/http/ExplainPlan.java
 create mode 100644 sql/src/test/java/org/apache/druid/sql/client/BrokerClientImplTest.java
 create mode 100644 sql/src/test/java/org/apache/druid/sql/guice/BrokerServiceModuleTest.java
 create mode 100644 sql/src/test/java/org/apache/druid/sql/http/ExplainPlanTest.java

diff --git a/server/src/main/java/org/apache/druid/discovery/BrokerClient.java b/server/src/main/java/org/apache/druid/discovery/BrokerClient.java
index bdee9b8dfe4a..a0ddbf42bed8 100644
--- a/server/src/main/java/org/apache/druid/discovery/BrokerClient.java
+++ b/server/src/main/java/org/apache/druid/discovery/BrokerClient.java
@@ -29,6 +29,7 @@
 import org.apache.druid.java.util.http.client.Request;
 import org.apache.druid.java.util.http.client.response.StringFullResponseHandler;
 import org.apache.druid.java.util.http.client.response.StringFullResponseHolder;
+import org.apache.druid.rpc.ServiceClient;
 import org.jboss.netty.channel.ChannelException;
 import org.jboss.netty.handler.codec.http.HttpMethod;
 import org.jboss.netty.handler.codec.http.HttpResponseStatus;
@@ -41,7 +42,10 @@
 
 /**
  * This class facilitates interaction with Broker.
+ * Note that this should be removed and reconciled with org.apache.druid.sql.client.BrokerClient, which has the
+ * built-in functionality of {@link ServiceClient}, and proper Guice and service discovery wired in.
  */
+@Deprecated
 public class BrokerClient
 {
   private static final int MAX_RETRIES = 5;
diff --git a/server/src/main/java/org/apache/druid/rpc/guice/ServiceClientModule.java b/server/src/main/java/org/apache/druid/rpc/guice/ServiceClientModule.java
index 51dd2b89d736..94dfeb29d957 100644
--- a/server/src/main/java/org/apache/druid/rpc/guice/ServiceClientModule.java
+++ b/server/src/main/java/org/apache/druid/rpc/guice/ServiceClientModule.java
@@ -47,8 +47,8 @@
 
 public class ServiceClientModule implements DruidModule
 {
+  public static final int CLIENT_MAX_ATTEMPTS = 6;
   private static final int CONNECT_EXEC_THREADS = 4;
-  private static final int CLIENT_MAX_ATTEMPTS = 6;
 
   @Override
   public void configure(Binder binder)
@@ -59,11 +59,9 @@ public void configure(Binder binder)
   @Provides
   @LazySingleton
   @EscalatedGlobal
-  public ServiceClientFactory makeServiceClientFactory(@EscalatedGlobal final HttpClient httpClient)
+  public ServiceClientFactory getServiceClientFactory(@EscalatedGlobal final HttpClient httpClient)
   {
-    final ScheduledExecutorService connectExec =
-        ScheduledExecutors.fixed(CONNECT_EXEC_THREADS, "ServiceClientFactory-%d");
-    return new ServiceClientFactoryImpl(httpClient, connectExec);
+    return makeServiceClientFactory(httpClient);
   }
 
   @Provides
@@ -117,4 +115,11 @@ public CoordinatorClient makeCoordinatorClient(
         jsonMapper
     );
   }
+
+  public static ServiceClientFactory makeServiceClientFactory(@EscalatedGlobal final HttpClient httpClient)
+  {
+    final ScheduledExecutorService connectExec =
+        ScheduledExecutors.fixed(CONNECT_EXEC_THREADS, "ServiceClientFactory-%d");
+    return new ServiceClientFactoryImpl(httpClient, connectExec);
+  }
 }
diff --git a/server/src/test/java/org/apache/druid/rpc/guice/ServiceClientModuleTest.java b/server/src/test/java/org/apache/druid/rpc/guice/ServiceClientModuleTest.java
new file mode 100644
index 000000000000..5d8a07d20a8a
--- /dev/null
+++ b/server/src/test/java/org/apache/druid/rpc/guice/ServiceClientModuleTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.rpc.guice;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.druid.client.coordinator.CoordinatorClient;
+import org.apache.druid.discovery.DruidNodeDiscoveryProvider;
+import org.apache.druid.guice.DruidGuiceExtensions;
+import org.apache.druid.guice.LifecycleModule;
+import org.apache.druid.guice.annotations.EscalatedGlobal;
+import org.apache.druid.jackson.JacksonModule;
+import org.apache.druid.java.util.http.client.HttpClient;
+import org.apache.druid.rpc.ServiceClientFactory;
+import org.apache.druid.rpc.ServiceLocator;
+import org.apache.druid.rpc.indexing.OverlordClient;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+
+import static org.junit.Assert.assertNotNull;
+
+public class ServiceClientModuleTest
+{
+  private Injector injector;
+
+  @Rule
+  public MockitoRule mockitoRule = MockitoJUnit.rule();
+
+  @Mock
+  private HttpClient httpClient;
+
+  @Mock
+  private DruidNodeDiscoveryProvider discoveryProvider;
+
+  @Mock
+  private ServiceLocator serviceLocator;
+
+  @Mock
+  private ServiceClientFactory serviceClientFactory;
+
+  @Before
+  public void setUp()
+  {
+    injector = Guice.createInjector(
+        ImmutableList.of(
+            new DruidGuiceExtensions(),
+            new LifecycleModule(),
+          new JacksonModule(),
+          new ServiceClientModule(),
+          binder -> {
+            binder.bind(HttpClient.class).annotatedWith(EscalatedGlobal.class).toInstance(httpClient);
+            binder.bind(ServiceLocator.class).toInstance(serviceLocator);
+            binder.bind(DruidNodeDiscoveryProvider.class).toInstance(discoveryProvider);
+            binder.bind(ServiceClientFactory.class).toInstance(serviceClientFactory);
+          }
+          )
+    );
+  }
+
+  @Test
+  public void testGetServiceClientFactory()
+  {
+    assertNotNull(injector.getInstance(ServiceClientFactory.class));
+  }
+
+  @Test
+  public void testGetOverlordClient()
+  {
+    assertNotNull(injector.getInstance(OverlordClient.class));
+  }
+
+  @Test
+  public void testGetCoordinatorClient()
+  {
+    assertNotNull(injector.getInstance(CoordinatorClient.class));
+  }
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ExplainAttributes.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ExplainAttributes.java
index e2ae4fa7a10c..533de7d58f2f 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/ExplainAttributes.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/ExplainAttributes.java
@@ -19,12 +19,14 @@
 
 package org.apache.druid.sql.calcite.planner;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.druid.java.util.common.granularity.Granularity;
 
 import javax.annotation.Nullable;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * ExplainAttributes holds the attributes of a SQL statement that is used in the EXPLAIN PLAN result.
@@ -45,6 +47,7 @@ public final class ExplainAttributes
   @Nullable
   private final String replaceTimeChunks;
 
+  @JsonCreator
   public ExplainAttributes(
       @JsonProperty("statementType") final String statementType,
       @JsonProperty("targetDataSource") @Nullable final String targetDataSource,
@@ -117,6 +120,29 @@ public String getReplaceTimeChunks()
     return replaceTimeChunks;
   }
 
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    ExplainAttributes that = (ExplainAttributes) o;
+    return Objects.equals(statementType, that.statementType)
+           && Objects.equals(targetDataSource, that.targetDataSource)
+           && Objects.equals(partitionedBy, that.partitionedBy)
+           && Objects.equals(clusteredBy, that.clusteredBy)
+           && Objects.equals(replaceTimeChunks, that.replaceTimeChunks);
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hash(statementType, targetDataSource, partitionedBy, clusteredBy, replaceTimeChunks);
+  }
+
   @Override
   public String toString()
   {
diff --git a/sql/src/main/java/org/apache/druid/sql/client/Broker.java b/sql/src/main/java/org/apache/druid/sql/client/Broker.java
new file mode 100644
index 000000000000..fb20c5166c8f
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/client/Broker.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.client;
+
+import com.google.inject.BindingAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@BindingAnnotation
+@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface Broker
+{
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/client/BrokerClient.java b/sql/src/main/java/org/apache/druid/sql/client/BrokerClient.java
new file mode 100644
index 000000000000..14cbbb7bff6a
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/client/BrokerClient.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.client;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.druid.sql.http.ExplainPlan;
+import org.apache.druid.sql.http.SqlQuery;
+import org.apache.druid.sql.http.SqlTaskStatus;
+
+import java.util.List;
+
+/**
+ * High-level Broker client.
+ * <p>
+ * All methods return futures, enabling asynchronous logic. If you want a synchronous response, use
+ * {@code FutureUtils.get} or {@code FutureUtils.getUnchecked}.
+ * Futures resolve to exceptions in the manner described by {@link org.apache.druid.rpc.ServiceClient#asyncRequest}.
+ * </p>
+ * Typically acquired via Guice, where it is registered using {@link org.apache.druid.rpc.guice.ServiceClientModule}.
+ */
+public interface BrokerClient
+{
+  /**
+   * Submit the given {@code sqlQuery} to the Broker's SQL task endpoint.
+   */
+  ListenableFuture<SqlTaskStatus> submitSqlTask(SqlQuery sqlQuery);
+
+  /**
+   * Fetches the explain plan for the given {@code sqlQuery} from the Broker's SQL task endpoint.
+   *
+   * @param sqlQuery the SQL query for which the {@code EXPLAIN PLAN FOR} information is to be fetched
+   */
+  ListenableFuture<List<ExplainPlan>> fetchExplainPlan(SqlQuery sqlQuery);
+}
diff --git a/sql/src/main/java/org/apache/druid/sql/client/BrokerClientImpl.java b/sql/src/main/java/org/apache/druid/sql/client/BrokerClientImpl.java
new file mode 100644
index 000000000000..b3e064341e63
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/client/BrokerClientImpl.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.client;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.druid.common.guava.FutureUtils;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.jackson.JacksonUtils;
+import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler;
+import org.apache.druid.rpc.RequestBuilder;
+import org.apache.druid.rpc.ServiceClient;
+import org.apache.druid.sql.http.ExplainPlan;
+import org.apache.druid.sql.http.SqlQuery;
+import org.apache.druid.sql.http.SqlTaskStatus;
+import org.jboss.netty.handler.codec.http.HttpMethod;
+
+import java.util.List;
+
+public class BrokerClientImpl implements BrokerClient
+{
+  private final ServiceClient client;
+  private final ObjectMapper jsonMapper;
+
+  public BrokerClientImpl(final ServiceClient client, final ObjectMapper jsonMapper)
+  {
+    this.client = client;
+    this.jsonMapper = jsonMapper;
+  }
+
+  @Override
+  public ListenableFuture<SqlTaskStatus> submitSqlTask(final SqlQuery sqlQuery)
+  {
+    return FutureUtils.transform(
+        client.asyncRequest(
+            new RequestBuilder(HttpMethod.POST, "/druid/v2/sql/task/")
+                .jsonContent(jsonMapper, sqlQuery),
+            new BytesFullResponseHandler()
+        ),
+        holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), SqlTaskStatus.class)
+    );
+  }
+
+  @Override
+  public ListenableFuture<List<ExplainPlan>> fetchExplainPlan(final SqlQuery sqlQuery)
+  {
+    final SqlQuery explainSqlQuery = new SqlQuery(
+        StringUtils.format("EXPLAIN PLAN FOR %s", sqlQuery.getQuery()),
+        null,
+        false,
+        false,
+        false,
+        null,
+        null
+    );
+    return FutureUtils.transform(
+        client.asyncRequest(
+            new RequestBuilder(HttpMethod.POST, "/druid/v2/sql/task/")
+                .jsonContent(jsonMapper, explainSqlQuery),
+            new BytesFullResponseHandler()
+        ),
+        holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference<List<ExplainPlan>>() {})
+    );
+  }
+}
+
diff --git a/sql/src/main/java/org/apache/druid/sql/guice/BrokerServiceModule.java b/sql/src/main/java/org/apache/druid/sql/guice/BrokerServiceModule.java
new file mode 100644
index 000000000000..05e022f83107
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/guice/BrokerServiceModule.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.guice;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.inject.Binder;
+import com.google.inject.Provides;
+import org.apache.druid.discovery.DruidNodeDiscoveryProvider;
+import org.apache.druid.discovery.NodeRole;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.guice.ManageLifecycle;
+import org.apache.druid.guice.annotations.EscalatedGlobal;
+import org.apache.druid.guice.annotations.Json;
+import org.apache.druid.initialization.DruidModule;
+import org.apache.druid.java.util.http.client.HttpClient;
+import org.apache.druid.rpc.DiscoveryServiceLocator;
+import org.apache.druid.rpc.ServiceClientFactory;
+import org.apache.druid.rpc.ServiceLocator;
+import org.apache.druid.rpc.StandardRetryPolicy;
+import org.apache.druid.rpc.guice.ServiceClientModule;
+import org.apache.druid.sql.client.Broker;
+import org.apache.druid.sql.client.BrokerClient;
+import org.apache.druid.sql.client.BrokerClientImpl;
+
+/**
+ * Module that processes can install if they require a {@link BrokerClient}.
+ * <p>
+ * Similar to {@link ServiceClientModule}, but since {@link BrokerClient} depends
+ * on classes from the sql module, this is a separate module within the sql package.
+ * </p>
+ */
+public class BrokerServiceModule implements DruidModule
+{
+  @Override
+  public void configure(Binder binder)
+  {
+    // Nothing to do.
+  }
+
+  @Provides
+  @LazySingleton
+  @EscalatedGlobal
+  public ServiceClientFactory getServiceClientFactory(@EscalatedGlobal final HttpClient httpClient)
+  {
+    return ServiceClientModule.makeServiceClientFactory(httpClient);
+  }
+
+  @Provides
+  @ManageLifecycle
+  @Broker
+  public ServiceLocator makeBrokerServiceLocator(final DruidNodeDiscoveryProvider discoveryProvider)
+  {
+    return new DiscoveryServiceLocator(discoveryProvider, NodeRole.BROKER);
+  }
+
+  @Provides
+  @LazySingleton
+  public BrokerClient makeBrokerClient(
+      @Json final ObjectMapper jsonMapper,
+      @EscalatedGlobal final ServiceClientFactory clientFactory,
+      @Broker final ServiceLocator serviceLocator
+  )
+  {
+    return new BrokerClientImpl(
+        clientFactory.makeClient(
+            NodeRole.BROKER.getJsonName(),
+            serviceLocator,
+            StandardRetryPolicy.builder().maxAttempts(ServiceClientModule.CLIENT_MAX_ATTEMPTS).build()
+        ),
+        jsonMapper
+    );
+  }
+}
+
diff --git a/sql/src/main/java/org/apache/druid/sql/http/ExplainPlan.java b/sql/src/main/java/org/apache/druid/sql/http/ExplainPlan.java
new file mode 100644
index 000000000000..68defc4b2a4d
--- /dev/null
+++ b/sql/src/main/java/org/apache/druid/sql/http/ExplainPlan.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.http;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import org.apache.druid.sql.calcite.planner.ExplainAttributes;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * Class that encapsulates the information of a single plan for an {@code EXPLAIN PLAN FOR} query.
+ * <p>
+ * Similar to {@link #getAttributes()}, it's possible to provide more structure to {@link #getPlan()},
+ * at least for the native query explain, but there's currently no use case for it.
+ * </p>
+ */
+public class ExplainPlan
+{
+  @JsonProperty("PLAN")
+  private final String plan;
+
+  @JsonProperty("RESOURCES")
+  private final String resources;
+
+  @JsonProperty("ATTRIBUTES")
+  @JsonDeserialize(using = ExplainAttributesDeserializer.class)
+  private final ExplainAttributes attributes;
+
+  @JsonCreator
+  public ExplainPlan(
+      @JsonProperty("PLAN") final String plan,
+      @JsonProperty("RESOURCES") final String resources,
+      @JsonProperty("ATTRIBUTES") final ExplainAttributes attributes
+  )
+  {
+    this.plan = plan;
+    this.resources = resources;
+    this.attributes = attributes;
+  }
+
+  public String getPlan()
+  {
+    return plan;
+  }
+
+  public String getResources()
+  {
+    return resources;
+  }
+
+  public ExplainAttributes getAttributes()
+  {
+    return attributes;
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    ExplainPlan that = (ExplainPlan) o;
+    return Objects.equals(plan, that.plan)
+           && Objects.equals(resources, that.resources)
+           && Objects.equals(attributes, that.attributes);
+  }
+
+  @Override
+  public int hashCode()
+  {
+    return Objects.hash(plan, resources, attributes);
+  }
+
+  /**
+   * Custom deserializer for {@link ExplainAttributes} because the value for {@link #attributes} in the plan
+   * is encoded as a JSON string. This deserializer tells Jackson on how to parse the JSON string
+   * and map it to the fields in the {@link ExplainAttributes} class.
+   */
+  private static class ExplainAttributesDeserializer extends JsonDeserializer<ExplainAttributes>
+  {
+    @Override
+    public ExplainAttributes deserialize(JsonParser jsonParser, DeserializationContext context) throws IOException
+    {
+      final ObjectMapper objectMapper = (ObjectMapper) jsonParser.getCodec();
+      return objectMapper.readValue(jsonParser.getText(), ExplainAttributes.class);
+    }
+  }
+}
+
+
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/ExplainAttributesTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/ExplainAttributesTest.java
index 53e2abf2749d..d203dd340028 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/ExplainAttributesTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/ExplainAttributesTest.java
@@ -19,8 +19,8 @@
 
 package org.apache.druid.sql.calcite.planner;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.druid.error.DruidException;
 import org.apache.druid.jackson.DefaultObjectMapper;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.junit.Assert;
@@ -28,14 +28,16 @@
 
 import java.util.Arrays;
 
+import static org.junit.Assert.assertEquals;
+
 public class ExplainAttributesTest
 {
-  private static final ObjectMapper DEFAULT_OBJECT_MAPPER = new DefaultObjectMapper();
+  private static final ObjectMapper MAPPER = new DefaultObjectMapper();
 
   @Test
-  public void testSimpleGetters()
+  public void testGetters()
   {
-    ExplainAttributes selectAttributes = new ExplainAttributes("SELECT", null, null, null, null);
+    final ExplainAttributes selectAttributes = new ExplainAttributes("SELECT", null, null, null, null);
     Assert.assertEquals("SELECT", selectAttributes.getStatementType());
     Assert.assertNull(selectAttributes.getTargetDataSource());
     Assert.assertNull(selectAttributes.getPartitionedBy());
@@ -44,9 +46,9 @@ public void testSimpleGetters()
   }
 
   @Test
-  public void testSerializeSelectAttributes() throws JsonProcessingException
+  public void testSerdeOfSelectAttributes()
   {
-    ExplainAttributes selectAttributes = new ExplainAttributes(
+    final ExplainAttributes selectAttributes = new ExplainAttributes(
         "SELECT",
         null,
         null,
@@ -56,13 +58,14 @@ public void testSerializeSelectAttributes() throws JsonProcessingException
     final String expectedAttributes = "{"
                                       + "\"statementType\":\"SELECT\""
                                       + "}";
-    Assert.assertEquals(expectedAttributes, DEFAULT_OBJECT_MAPPER.writeValueAsString(selectAttributes));
+
+    testSerde(selectAttributes, expectedAttributes);
   }
 
   @Test
-  public void testSerializeInsertAttributes() throws JsonProcessingException
+  public void testSerdeOfInsertAttributes()
   {
-    ExplainAttributes insertAttributes = new ExplainAttributes(
+    final ExplainAttributes insertAttributes = new ExplainAttributes(
         "INSERT",
         "foo",
         Granularities.DAY,
@@ -74,13 +77,13 @@ public void testSerializeInsertAttributes() throws JsonProcessingException
                                       + "\"targetDataSource\":\"foo\","
                                       + "\"partitionedBy\":\"DAY\""
                                       + "}";
-    Assert.assertEquals(expectedAttributes, DEFAULT_OBJECT_MAPPER.writeValueAsString(insertAttributes));
+    testSerde(insertAttributes, expectedAttributes);
   }
 
   @Test
-  public void testSerializeInsertAllAttributes() throws JsonProcessingException
+  public void testSerdeOfInsertAllAttributes()
   {
-    ExplainAttributes insertAttributes = new ExplainAttributes(
+    final ExplainAttributes insertAttributes = new ExplainAttributes(
         "INSERT",
         "foo",
         Granularities.ALL,
@@ -92,78 +95,100 @@ public void testSerializeInsertAllAttributes() throws JsonProcessingException
                                       + "\"targetDataSource\":\"foo\","
                                       + "\"partitionedBy\":{\"type\":\"all\"}"
                                       + "}";
-    Assert.assertEquals(expectedAttributes, DEFAULT_OBJECT_MAPPER.writeValueAsString(insertAttributes));
+    testSerde(insertAttributes, expectedAttributes);
   }
 
   @Test
-  public void testSerializeReplaceAttributes() throws JsonProcessingException
+  public void testSerdeOfReplaceAttributes()
   {
-    ExplainAttributes replaceAttributes1 = new ExplainAttributes(
+    final ExplainAttributes replaceAttributes = new ExplainAttributes(
         "REPLACE",
         "foo",
         Granularities.HOUR,
         null,
         "ALL"
     );
-    final String expectedAttributes1 = "{"
+    final String expectedAttributes = "{"
                                        + "\"statementType\":\"REPLACE\","
                                        + "\"targetDataSource\":\"foo\","
                                        + "\"partitionedBy\":\"HOUR\","
                                        + "\"replaceTimeChunks\":\"ALL\""
                                        + "}";
-    Assert.assertEquals(expectedAttributes1, DEFAULT_OBJECT_MAPPER.writeValueAsString(replaceAttributes1));
+    testSerde(replaceAttributes, expectedAttributes);
 
+  }
 
-    ExplainAttributes replaceAttributes2 = new ExplainAttributes(
+  @Test
+  public void testSerdeOfReplaceAttributesWithTimeChunks()
+  {
+    final ExplainAttributes replaceAttributes = new ExplainAttributes(
         "REPLACE",
         "foo",
         Granularities.HOUR,
         null,
         "2019-08-25T02:00:00.000Z/2019-08-25T03:00:00.000Z"
     );
-    final String expectedAttributes2 = "{"
+    final String expectedAttributes = "{"
                                        + "\"statementType\":\"REPLACE\","
                                        + "\"targetDataSource\":\"foo\","
                                        + "\"partitionedBy\":\"HOUR\","
                                        + "\"replaceTimeChunks\":\"2019-08-25T02:00:00.000Z/2019-08-25T03:00:00.000Z\""
                                        + "}";
-    Assert.assertEquals(expectedAttributes2, DEFAULT_OBJECT_MAPPER.writeValueAsString(replaceAttributes2));
+    testSerde(replaceAttributes, expectedAttributes);
   }
 
   @Test
-  public void testSerializeReplaceWithClusteredByAttributes() throws JsonProcessingException
+  public void testReplaceAttributesWithClusteredBy()
   {
-    ExplainAttributes replaceAttributes1 = new ExplainAttributes(
+    final ExplainAttributes replaceAttributes = new ExplainAttributes(
         "REPLACE",
         "foo",
         Granularities.HOUR,
         Arrays.asList("foo", "CEIL(`f2`)"),
         "ALL"
     );
-    final String expectedAttributes1 = "{"
+    final String expectedAttributes = "{"
                                        + "\"statementType\":\"REPLACE\","
                                        + "\"targetDataSource\":\"foo\","
                                        + "\"partitionedBy\":\"HOUR\","
                                        + "\"clusteredBy\":[\"foo\",\"CEIL(`f2`)\"],"
                                        + "\"replaceTimeChunks\":\"ALL\""
                                        + "}";
-    Assert.assertEquals(expectedAttributes1, DEFAULT_OBJECT_MAPPER.writeValueAsString(replaceAttributes1));
-
+    testSerde(replaceAttributes, expectedAttributes);
+  }
 
-    ExplainAttributes replaceAttributes2 = new ExplainAttributes(
+  @Test
+  public void testReplaceAttributesWithClusteredByAndTimeChunks()
+  {
+    final ExplainAttributes replaceAttributes = new ExplainAttributes(
         "REPLACE",
         "foo",
         Granularities.HOUR,
         Arrays.asList("foo", "boo"),
         "2019-08-25T02:00:00.000Z/2019-08-25T03:00:00.000Z"
     );
-    final String expectedAttributes2 = "{"
+    final String expectedAttributes = "{"
                                        + "\"statementType\":\"REPLACE\","
                                        + "\"targetDataSource\":\"foo\","
                                        + "\"partitionedBy\":\"HOUR\","
                                        + "\"clusteredBy\":[\"foo\",\"boo\"],"
                                        + "\"replaceTimeChunks\":\"2019-08-25T02:00:00.000Z/2019-08-25T03:00:00.000Z\""
                                        + "}";
-    Assert.assertEquals(expectedAttributes2, DEFAULT_OBJECT_MAPPER.writeValueAsString(replaceAttributes2));
+    testSerde(replaceAttributes, expectedAttributes);
   }
+
+  private void testSerde(final ExplainAttributes explainAttributes, final String expectedSerializedAttributes)
+  {
+    final ExplainAttributes observedAttributes;
+    try {
+      final String observedSerializedAttributes = MAPPER.writeValueAsString(explainAttributes);
+      assertEquals(expectedSerializedAttributes, observedSerializedAttributes);
+      observedAttributes = MAPPER.readValue(observedSerializedAttributes, ExplainAttributes.class);
+    }
+    catch (Exception e) {
+      throw DruidException.defensive(e, "Error serializing/deserializing explain plan[%s].", explainAttributes);
+    }
+    assertEquals(explainAttributes, observedAttributes);
+  }
+
 }
diff --git a/sql/src/test/java/org/apache/druid/sql/client/BrokerClientImplTest.java b/sql/src/test/java/org/apache/druid/sql/client/BrokerClientImplTest.java
new file mode 100644
index 000000000000..51d66f03816d
--- /dev/null
+++ b/sql/src/test/java/org/apache/druid/sql/client/BrokerClientImplTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.client;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.calcite.avatica.SqlType;
+import org.apache.druid.indexer.TaskState;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.rpc.MockServiceClient;
+import org.apache.druid.rpc.RequestBuilder;
+import org.apache.druid.sql.calcite.planner.ExplainAttributes;
+import org.apache.druid.sql.http.ExplainPlan;
+import org.apache.druid.sql.http.ResultFormat;
+import org.apache.druid.sql.http.SqlParameter;
+import org.apache.druid.sql.http.SqlQuery;
+import org.apache.druid.sql.http.SqlTaskStatus;
+import org.jboss.netty.handler.codec.http.HttpMethod;
+import org.jboss.netty.handler.codec.http.HttpResponseStatus;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class BrokerClientImplTest
+{
+  private ObjectMapper jsonMapper;
+  private MockServiceClient serviceClient;
+  private BrokerClient brokerClient;
+
+  @Before
+  public void setup()
+  {
+    jsonMapper = new DefaultObjectMapper();
+    serviceClient = new MockServiceClient();
+    brokerClient = new BrokerClientImpl(serviceClient, jsonMapper);
+  }
+
+  @After
+  public void tearDown()
+  {
+    serviceClient.verify();
+  }
+
+  @Test
+  public void testSubmitSqlTask() throws Exception
+  {
+    final SqlQuery query = new SqlQuery(
+        "REPLACE INTO foo OVERWRITE ALL SELECT * FROM bar PARTITIONED BY ALL",
+        ResultFormat.ARRAY,
+        true,
+        true,
+        true,
+        ImmutableMap.of("useCache", false),
+        ImmutableList.of(new SqlParameter(SqlType.INTEGER, 1))
+    );
+    final SqlTaskStatus taskStatus = new SqlTaskStatus("taskId1", TaskState.RUNNING, null);
+
+    serviceClient.expectAndRespond(
+        new RequestBuilder(HttpMethod.POST, "/druid/v2/sql/task/")
+            .jsonContent(jsonMapper, query),
+        HttpResponseStatus.OK,
+        ImmutableMap.of(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON),
+        jsonMapper.writeValueAsBytes(taskStatus)
+    );
+
+    assertEquals(taskStatus, brokerClient.submitSqlTask(query).get());
+  }
+
+  @Test
+  public void testFetchExplainPlan() throws Exception
+  {
+    final SqlQuery query = new SqlQuery(
+        "REPLACE INTO foo OVERWRITE ALL SELECT * FROM bar PARTITIONED BY ALL",
+        ResultFormat.ARRAY,
+        true,
+        true,
+        true,
+        ImmutableMap.of("useCache", false),
+        ImmutableList.of(new SqlParameter(SqlType.INTEGER, 1))
+    );
+    final SqlQuery explainQuery = new SqlQuery(
+        StringUtils.format("EXPLAIN PLAN FOR %s", query.getQuery()),
+        null,
+        false,
+        false,
+        false,
+        null,
+        null
+    );
+
+    final String plan = "[{\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"external\",\"inputSource\":{\"type\":\"inline\",\"data\":\"a,b,1\\nc,d,2\\n\"},\"inputFormat\":{\"type\":\"csv\",\"columns\":[\"x\",\"y\",\"z\"]},\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"x\",\"y\",\"z\"],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\",\"sqlReplaceTimeChunks\":\"all\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"columnTypes\":[\"STRING\",\"STRING\",\"LONG\"],\"granularity\":{\"type\":\"all\"},\"legacy\":false},\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}],\"columnMappings\":[{\"queryColumn\":\"x\",\"outputColumn\":\"x\"},{\"queryColumn\":\"y\",\"outputColumn\":\"y\"},{\"queryColumn\":\"z\",\"outputColumn\":\"z\"}]}]";
+    final String resources = "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]";
+    final ExplainAttributes attributes = new ExplainAttributes("REPLACE", "foo", Granularities.ALL, null, "all");
+
+    final List<Map<String, Object>> givenPlans = ImmutableList.of(
+        ImmutableMap.of(
+            "PLAN",
+            plan,
+            "RESOURCES",
+            resources,
+            "ATTRIBUTES",
+            jsonMapper.writeValueAsString(attributes)
+        )
+    );
+
+    serviceClient.expectAndRespond(
+        new RequestBuilder(HttpMethod.POST, "/druid/v2/sql/task/")
+            .jsonContent(jsonMapper, explainQuery),
+        HttpResponseStatus.OK,
+        ImmutableMap.of(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON),
+        jsonMapper.writeValueAsBytes(givenPlans)
+    );
+
+    assertEquals(
+        ImmutableList.of(new ExplainPlan(plan, resources, attributes)),
+        brokerClient.fetchExplainPlan(query).get()
+    );
+  }
+
+}
diff --git a/sql/src/test/java/org/apache/druid/sql/guice/BrokerServiceModuleTest.java b/sql/src/test/java/org/apache/druid/sql/guice/BrokerServiceModuleTest.java
new file mode 100644
index 000000000000..9b59ad86733e
--- /dev/null
+++ b/sql/src/test/java/org/apache/druid/sql/guice/BrokerServiceModuleTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.guice;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.ConfigurationException;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.druid.client.coordinator.CoordinatorClient;
+import org.apache.druid.discovery.DruidNodeDiscoveryProvider;
+import org.apache.druid.guice.DruidGuiceExtensions;
+import org.apache.druid.guice.LifecycleModule;
+import org.apache.druid.guice.annotations.EscalatedGlobal;
+import org.apache.druid.jackson.JacksonModule;
+import org.apache.druid.java.util.http.client.HttpClient;
+import org.apache.druid.rpc.ServiceClientFactory;
+import org.apache.druid.rpc.ServiceLocator;
+import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.sql.client.BrokerClient;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnit;
+import org.mockito.junit.MockitoRule;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
+
+public class BrokerServiceModuleTest
+{
+  private Injector injector;
+
+  @Rule
+  public MockitoRule mockitoRule = MockitoJUnit.rule();
+
+  @Mock
+  private HttpClient httpClient;
+
+  @Mock
+  private DruidNodeDiscoveryProvider discoveryProvider;
+
+  @Mock
+  private ServiceLocator serviceLocator;
+
+  @Mock
+  private ServiceClientFactory serviceClientFactory;
+
+  @Before
+  public void setUp()
+  {
+    injector = Guice.createInjector(
+        ImmutableList.of(
+            new DruidGuiceExtensions(),
+            new LifecycleModule(),
+            new JacksonModule(),
+            new BrokerServiceModule(),
+            binder -> {
+              binder.bind(HttpClient.class).annotatedWith(EscalatedGlobal.class).toInstance(httpClient);
+              binder.bind(ServiceLocator.class).toInstance(serviceLocator);
+              binder.bind(DruidNodeDiscoveryProvider.class).toInstance(discoveryProvider);
+              binder.bind(ServiceClientFactory.class).toInstance(serviceClientFactory);
+            }
+        )
+    );
+  }
+
+  @Test
+  public void testGetServiceClientFactory()
+  {
+    assertNotNull(injector.getInstance(ServiceClientFactory.class));
+  }
+
+  @Test
+  public void testGetBrokerClient()
+  {
+    assertNotNull(injector.getInstance(BrokerClient.class));
+  }
+
+  @Test
+  public void testGetCoordinatorClient()
+  {
+    assertThrows(
+        ConfigurationException.class,
+        () -> injector.getInstance(CoordinatorClient.class)
+    );
+  }
+
+  @Test
+  public void testGetOverlordClient()
+  {
+    assertThrows(
+        ConfigurationException.class,
+        () -> injector.getInstance(OverlordClient.class)
+    );
+  }
+}
diff --git a/sql/src/test/java/org/apache/druid/sql/http/ExplainPlanTest.java b/sql/src/test/java/org/apache/druid/sql/http/ExplainPlanTest.java
new file mode 100644
index 000000000000..e3385fc5f518
--- /dev/null
+++ b/sql/src/test/java/org/apache/druid/sql/http/ExplainPlanTest.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.sql.http;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.sql.calcite.planner.ExplainAttributes;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class ExplainPlanTest
+{
+  private static final ObjectMapper MAPPER = new DefaultObjectMapper();
+
+  @Test
+  public void testExplainPlanSerdeForSelectQuery() throws JsonProcessingException
+  {
+    final String plan = "[{\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"inline\",\"columnNames\":[\"EXPR$0\"],\"columnTypes\":[\"LONG\"],\"rows\":[[2]]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"EXPR$0\"],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"columnTypes\":[\"LONG\"],\"granularity\":{\"type\":\"all\"},\"legacy\":false},\"signature\":[{\"name\":\"EXPR$0\",\"type\":\"LONG\"}],\"columnMappings\":[{\"queryColumn\":\"EXPR$0\",\"outputColumn\":\"EXPR$0\"}]}]";
+    final String resources = "[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+    final ExplainAttributes attributes = new ExplainAttributes("SELECT", null, null, null, null);
+
+    final List<Map<String, Object>> givenPlans = ImmutableList.of(
+        ImmutableMap.of(
+            "PLAN",
+            plan,
+            "RESOURCES",
+            resources,
+            "ATTRIBUTES",
+            MAPPER.writeValueAsString(attributes)
+        )
+    );
+
+    testSerde(givenPlans, ImmutableList.of(new ExplainPlan(plan, resources, attributes)));
+  }
+
+  @Test
+  public void testExplainPlanSerdeForReplaceQuery() throws JsonProcessingException
+  {
+    final String plan = "[{\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"external\",\"inputSource\":{\"type\":\"inline\",\"data\":\"a,b,1\\nc,d,2\\n\"},\"inputFormat\":{\"type\":\"csv\",\"columns\":[\"x\",\"y\",\"z\"]},\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"x\",\"y\",\"z\"],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\",\"sqlReplaceTimeChunks\":\"all\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"columnTypes\":[\"STRING\",\"STRING\",\"LONG\"],\"granularity\":{\"type\":\"all\"},\"legacy\":false},\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}],\"columnMappings\":[{\"queryColumn\":\"x\",\"outputColumn\":\"x\"},{\"queryColumn\":\"y\",\"outputColumn\":\"y\"},{\"queryColumn\":\"z\",\"outputColumn\":\"z\"}]}]";
+    final String resources = "[{\"name\":\"EXTERNAL\",\"type\":\"EXTERNAL\"},{\"name\":\"dst\",\"type\":\"DATASOURCE\"}]";
+    final ExplainAttributes attributes = new ExplainAttributes("REPLACE", "dst", Granularities.ALL, null, "all");
+
+    final List<Map<String, Object>> givenPlans = ImmutableList.of(
+        ImmutableMap.of(
+            "PLAN",
+            plan,
+            "RESOURCES",
+            resources,
+            "ATTRIBUTES",
+            MAPPER.writeValueAsString(attributes)
+        )
+    );
+
+    testSerde(givenPlans, ImmutableList.of(new ExplainPlan(plan, resources, attributes)));
+  }
+
+  @Test
+  public void testExplainPlanSerdeForInsertQuery() throws JsonProcessingException
+  {
+    final String plan = "[{\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"external\",\"inputSource\":{\"type\":\"inline\",\"data\":\"a,b,1\\nc,d,2\\n\"},\"inputFormat\":{\"type\":\"csv\",\"columns\":[\"x\",\"y\",\"z\"]},\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}]},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"x\",\"y\",\"z\"],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlInsertSegmentGranularity\":\"{\\\"type\\\":\\\"all\\\"}\",\"sqlQueryId\":\"dummy\",\"sqlReplaceTimeChunks\":\"all\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"},\"columnTypes\":[\"STRING\",\"STRING\",\"LONG\"],\"granularity\":{\"type\":\"all\"},\"legacy\":false},\"signature\":[{\"name\":\"x\",\"type\":\"STRING\"},{\"name\":\"y\",\"type\":\"STRING\"},{\"name\":\"z\",\"type\":\"LONG\"}],\"columnMappings\":[{\"queryColumn\":\"x\",\"outputColumn\":\"x\"},{\"queryColumn\":\"y\",\"outputColumn\":\"y\"},{\"queryColumn\":\"z\",\"outputColumn\":\"z\"}]}]";
+    final String resources = "[{\"name\":\"dst\",\"type\":\"DATASOURCE\"},{\"name\":\"foo\",\"type\":\"DATASOURCE\"}]";
+    final ExplainAttributes attributes = new ExplainAttributes("INSERT", "foo", Granularities.DAY, ImmutableList.of("floor_m1", "dim1", "CEIL(\"m2\")"), null);
+
+    final List<Map<String, Object>> givenPlans = ImmutableList.of(
+        ImmutableMap.of(
+            "PLAN",
+            plan,
+            "RESOURCES",
+            resources,
+            "ATTRIBUTES",
+            MAPPER.writeValueAsString(attributes)
+        )
+    );
+
+    testSerde(givenPlans, ImmutableList.of(new ExplainPlan(plan, resources, attributes)));
+  }
+
+  private void testSerde(
+      final List<Map<String, Object>> givenPlans,
+      final List<ExplainPlan> expectedExplainPlans
+  )
+  {
+    final List<ExplainPlan> observedExplainPlans;
+    try {
+      observedExplainPlans = MAPPER.readValue(
+          MAPPER.writeValueAsString(givenPlans),
+          new TypeReference<List<ExplainPlan>>() {}
+      );
+    }
+    catch (Exception e) {
+      throw DruidException.defensive(e, "Error deserializing given plans[%s] into explain plans.", givenPlans);
+    }
+    assertEquals(expectedExplainPlans, observedExplainPlans);
+  }
+}

From 6cf372a7f46ed70c52f89fc49a3b912b2f9bfb27 Mon Sep 17 00:00:00 2001
From: Vadim Ogievetsky <vadim@ogievetsky.com>
Date: Mon, 21 Oct 2024 11:38:19 -0700
Subject: [PATCH 04/12] Web console: bump dependencies including d3 and
 typescript (#17381)

* bump deps including d3

* better clean script
---
 licenses.yaml                                 |  102 +-
 web-console/.gitignore                        |    2 +-
 web-console/jest.common.config.js             |   12 +-
 .../{jest.unit.config.js => jest.config.js}   |    0
 web-console/lib/keywords.d.ts                 |   22 -
 web-console/lib/{keywords.js => keywords.ts}  |    8 +-
 web-console/lib/sql-docs.d.ts                 |   20 -
 web-console/package-lock.json                 | 1700 ++++++++++-------
 web-console/package.json                      |   98 +-
 web-console/script/build                      |    4 +-
 web-console/script/clean                      |    6 +-
 ...create-sql-docs.js => create-sql-docs.mjs} |   21 +-
 web-console/src/ace-modes/dsql.ts             |   33 +-
 .../__snapshots__/menu-boolean.spec.tsx.snap  |    3 -
 .../segment-timeline.spec.tsx.snap            |    7 +
 .../suggestion-menu.spec.tsx.snap             |    1 -
 web-console/src/entry.tsx                     |    1 -
 web-console/src/setup-tests.ts                |    1 -
 .../views/explore-view/utils/table-query.ts   |    6 +-
 .../complex-menu-items.spec.tsx.snap          |    1 -
 .../number-menu-items.spec.tsx.snap           |    6 -
 .../string-menu-items.spec.tsx.snap           |    6 -
 .../time-menu-items.spec.tsx.snap             |    6 -
 .../flexible-query-input.tsx                  |    1 +
 .../run-panel/run-panel.spec.tsx              |   45 +-
 web-console/tsconfig.test.json                |    1 +
 .../{webpack.config.js => webpack.config.mjs} |   28 +-
 27 files changed, 1210 insertions(+), 931 deletions(-)
 rename web-console/{jest.unit.config.js => jest.config.js} (100%)
 delete mode 100644 web-console/lib/keywords.d.ts
 rename web-console/lib/{keywords.js => keywords.ts} (92%)
 delete mode 100644 web-console/lib/sql-docs.d.ts
 rename web-console/script/{create-sql-docs.js => create-sql-docs.mjs} (91%)
 rename web-console/{webpack.config.js => webpack.config.mjs} (88%)

diff --git a/licenses.yaml b/licenses.yaml
index a357b961643c..ea161c74b61a 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -5083,7 +5083,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Palantir Technologies
-version: 5.1.1
+version: 5.1.2
 
 ---
 
@@ -5092,7 +5092,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Palantir Technologies
-version: 5.10.5
+version: 5.13.1
 
 ---
 
@@ -5101,7 +5101,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Palantir Technologies
-version: 2.3.7
+version: 2.3.11
 
 ---
 
@@ -5110,7 +5110,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Palantir Technologies
-version: 5.3.7
+version: 5.3.11
 
 ---
 
@@ -5119,7 +5119,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Palantir Technologies
-version: 5.10.0
+version: 5.13.0
 
 ---
 
@@ -5128,7 +5128,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Palantir Technologies
-version: 5.2.1
+version: 5.2.5
 
 ---
 
@@ -5137,7 +5137,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Imply Data
-version: 0.22.22
+version: 0.22.23
 
 ---
 
@@ -5236,7 +5236,7 @@ license_category: binary
 module: web-console
 license_name: SIL Open Font License 1.1
 copyright: Google Inc.
-version: 5.0.28
+version: 5.1.0
 license_file_path: licenses/bin/@fontsource-open-sans.OFL
 
 ---
@@ -5286,27 +5286,17 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Microsoft Corporation.
-version: 18.2.42
+version: 18.3.11
 license_file_path: licenses/bin/@types-react.MIT
 
 ---
 
-name: "@types/scheduler"
-license_category: binary
-module: web-console
-license_name: MIT License
-copyright: Microsoft Corporation.
-version: 0.16.8
-license_file_path: licenses/bin/@types-scheduler.MIT
-
----
-
 name: "ace-builds"
 license_category: binary
 module: web-console
 license_name: BSD-3-Clause License
 copyright: Ajax.org B.V.
-version: 1.4.14
+version: 1.5.3
 license_file_path: licenses/bin/ace-builds.BSD3
 
 ---
@@ -5336,7 +5326,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Matt Zabriskie
-version: 1.7.4
+version: 1.7.7
 license_file_path: licenses/bin/axios.MIT
 
 ---
@@ -5475,7 +5465,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: TJ Holowaychuk
-version: 2.20.0
+version: 7.2.0
 license_file_path: licenses/bin/commander.MIT
 
 ---
@@ -5505,21 +5495,11 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: sudodoki
-version: 3.2.0
+version: 3.3.3
 license_file_path: licenses/bin/copy-to-clipboard.MIT
 
 ---
 
-name: "core-js"
-license_category: binary
-module: web-console
-license_name: MIT License
-copyright: Denis Pushkarev
-version: 3.10.1
-license_file_path: licenses/bin/core-js.MIT
-
----
-
 name: "cosmiconfig"
 license_category: binary
 module: web-console
@@ -5563,30 +5543,30 @@ license_file_path: licenses/bin/d3-array.BSD3
 name: "d3-axis"
 license_category: binary
 module: web-console
-license_name: BSD-3-Clause License
+license_name: ISC License
 copyright: Mike Bostock
-version: 2.1.0
-license_file_path: licenses/bin/d3-axis.BSD3
+version: 3.0.0
+license_file_path: licenses/bin/d3-axis.ISC
 
 ---
 
 name: "d3-color"
 license_category: binary
 module: web-console
-license_name: BSD-3-Clause License
+license_name: ISC License
 copyright: Mike Bostock
-version: 2.0.0
-license_file_path: licenses/bin/d3-color.BSD3
+version: 3.1.0
+license_file_path: licenses/bin/d3-color.ISC
 
 ---
 
 name: "d3-dsv"
 license_category: binary
 module: web-console
-license_name: BSD-3-Clause License
+license_name: ISC License
 copyright: Mike Bostock
-version: 2.0.0
-license_file_path: licenses/bin/d3-dsv.BSD3
+version: 3.0.1
+license_file_path: licenses/bin/d3-dsv.ISC
 
 ---
 
@@ -5603,10 +5583,10 @@ license_file_path: licenses/bin/d3-format.BSD3
 name: "d3-interpolate"
 license_category: binary
 module: web-console
-license_name: BSD-3-Clause License
+license_name: ISC License
 copyright: Mike Bostock
-version: 2.0.1
-license_file_path: licenses/bin/d3-interpolate.BSD3
+version: 3.0.1
+license_file_path: licenses/bin/d3-interpolate.ISC
 
 ---
 
@@ -5623,20 +5603,20 @@ license_file_path: licenses/bin/d3-scale-chromatic.ISC
 name: "d3-scale"
 license_category: binary
 module: web-console
-license_name: BSD-3-Clause License
+license_name: ISC License
 copyright: Mike Bostock
-version: 3.3.0
-license_file_path: licenses/bin/d3-scale.BSD3
+version: 4.0.2
+license_file_path: licenses/bin/d3-scale.ISC
 
 ---
 
 name: "d3-selection"
 license_category: binary
 module: web-console
-license_name: BSD-3-Clause License
+license_name: ISC License
 copyright: Mike Bostock
-version: 2.0.0
-license_file_path: licenses/bin/d3-selection.BSD3
+version: 3.0.0
+license_file_path: licenses/bin/d3-selection.ISC
 
 ---
 
@@ -5734,7 +5714,7 @@ license_category: binary
 module: web-console
 license_name: Apache License version 2.0
 copyright: Apache Software Foundation
-version: 5.4.3
+version: 5.5.1
 
 ---
 
@@ -5893,7 +5873,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Alexander Shtuchkin
-version: 0.4.24
+version: 0.6.3
 license_file_path: licenses/bin/iconv-lite.MIT
 
 ---
@@ -6062,7 +6042,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Alex Reardon
-version: 5.1.1
+version: 5.2.1
 license_file_path: licenses/bin/memoize-one.MIT
 
 ---
@@ -6291,7 +6271,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Facebook, Inc. and its affiliates.
-version: 18.2.0
+version: 18.3.1
 license_file_path: licenses/bin/react-dom.MIT
 
 ---
@@ -6401,7 +6381,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Facebook, Inc. and its affiliates.
-version: 18.2.0
+version: 18.3.1
 license_file_path: licenses/bin/react.MIT
 
 ---
@@ -6481,7 +6461,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Facebook, Inc. and its affiliates.
-version: 0.23.0
+version: 0.23.2
 license_file_path: licenses/bin/scheduler.MIT
 
 ---
@@ -6611,7 +6591,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Facebook, Inc. and its affiliates.
-version: 1.2.0
+version: 1.2.2
 license_file_path: licenses/bin/use-sync-external-store.MIT
 
 ---
@@ -6621,7 +6601,7 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Robert Kieffer and other contributors
-version: 7.0.2
+version: 7.0.3
 license_file_path: licenses/bin/uuid.MIT
 
 ---
@@ -6661,7 +6641,7 @@ license_category: binary
 module: web-console
 license_name: BSD-3-Clause License
 copyright: Baidu Inc.
-version: 5.4.4
+version: 5.6.0
 license_file_path: licenses/bin/zrender.BSD3
 
 ---
@@ -6671,6 +6651,6 @@ license_category: binary
 module: web-console
 license_name: MIT License
 copyright: Paul Henschel
-version: 4.3.2
+version: 4.5.5
 license_file_path: licenses/bin/zustand.MIT
 # Web console modules end
diff --git a/web-console/.gitignore b/web-console/.gitignore
index 2ca1b9d68199..8b06a648032d 100644
--- a/web-console/.gitignore
+++ b/web-console/.gitignore
@@ -10,7 +10,7 @@ coordinator-console/
 pages/
 /index.html
 
-lib/sql-docs.js
+lib/sql-docs.*
 
 .tscache
 tscommand-*.tmp.txt
diff --git a/web-console/jest.common.config.js b/web-console/jest.common.config.js
index 4c45547e6163..89e3dab5852b 100644
--- a/web-console/jest.common.config.js
+++ b/web-console/jest.common.config.js
@@ -16,12 +16,12 @@
  * limitations under the License.
  */
 
+const { createJsWithTsPreset } = require('ts-jest');
+
 module.exports = {
-  preset: 'ts-jest',
-  globals: {
-    'ts-jest': {
-      tsconfig: './tsconfig.test.json',
-    },
-  },
   testEnvironment: 'jsdom',
+  transformIgnorePatterns: ['/node_modules/(?!(d3-.+)/)'],
+  ...createJsWithTsPreset({
+    tsconfig: './tsconfig.test.json',
+  }),
 };
diff --git a/web-console/jest.unit.config.js b/web-console/jest.config.js
similarity index 100%
rename from web-console/jest.unit.config.js
rename to web-console/jest.config.js
diff --git a/web-console/lib/keywords.d.ts b/web-console/lib/keywords.d.ts
deleted file mode 100644
index f4bc7538c4b2..000000000000
--- a/web-console/lib/keywords.d.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-export const SQL_KEYWORDS: string[];
-export const SQL_EXPRESSION_PARTS: string[];
-export const SQL_CONSTANTS: string[];
-export const SQL_DYNAMICS: string[];
diff --git a/web-console/lib/keywords.js b/web-console/lib/keywords.ts
similarity index 92%
rename from web-console/lib/keywords.js
rename to web-console/lib/keywords.ts
index dbf10e12bac6..06d7ccdcc949 100644
--- a/web-console/lib/keywords.js
+++ b/web-console/lib/keywords.ts
@@ -18,7 +18,7 @@
 
 // Hand picked from https://druid.apache.org/docs/latest/querying/sql.html
 
-exports.SQL_KEYWORDS = [
+export const SQL_KEYWORDS = [
   'EXPLAIN PLAN FOR',
   'WITH',
   'AS',
@@ -75,7 +75,7 @@ exports.SQL_KEYWORDS = [
   'UNPIVOT',
 ];
 
-exports.SQL_EXPRESSION_PARTS = [
+export const SQL_EXPRESSION_PARTS = [
   'FILTER',
   'END',
   'ELSE',
@@ -114,9 +114,9 @@ exports.SQL_EXPRESSION_PARTS = [
   'CSV',
 ];
 
-exports.SQL_CONSTANTS = ['NULL', 'FALSE', 'TRUE'];
+export const SQL_CONSTANTS = ['NULL', 'FALSE', 'TRUE'];
 
-exports.SQL_DYNAMICS = [
+export const SQL_DYNAMICS = [
   'CURRENT_TIMESTAMP',
   'CURRENT_DATE',
   'LOCALTIME',
diff --git a/web-console/lib/sql-docs.d.ts b/web-console/lib/sql-docs.d.ts
deleted file mode 100644
index 5948206a2208..000000000000
--- a/web-console/lib/sql-docs.d.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-export const SQL_DATA_TYPES: Record<string, [runtime: string, description: string]>;
-export const SQL_FUNCTIONS: Record<string, [args: string, description: string][]>;
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index 6448c8084843..3431d44e4a51 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -9,72 +9,71 @@
       "version": "32.0.0",
       "license": "Apache-2.0",
       "dependencies": {
-        "@blueprintjs/core": "^5.10.5",
-        "@blueprintjs/datetime": "^5.3.6",
-        "@blueprintjs/datetime2": "^2.3.7",
-        "@blueprintjs/icons": "^5.10.0",
-        "@blueprintjs/select": "^5.2.1",
-        "@druid-toolkit/query": "^0.22.22",
-        "@fontsource/open-sans": "^5.0.28",
-        "ace-builds": "~1.4.14",
-        "axios": "^1.7.4",
+        "@blueprintjs/core": "^5.13.1",
+        "@blueprintjs/datetime": "^5.3.11",
+        "@blueprintjs/datetime2": "^2.3.11",
+        "@blueprintjs/icons": "^5.13.0",
+        "@blueprintjs/select": "^5.2.5",
+        "@druid-toolkit/query": "^0.22.23",
+        "@fontsource/open-sans": "^5.0.30",
+        "ace-builds": "~1.5.3",
+        "axios": "^1.7.7",
         "chronoshift": "^0.10.0",
         "classnames": "^2.2.6",
-        "copy-to-clipboard": "^3.2.0",
-        "core-js": "^3.10.1",
-        "d3-array": "^2.12.1",
-        "d3-axis": "^2.1.0",
-        "d3-dsv": "^2.0.0",
-        "d3-scale": "^3.3.0",
+        "copy-to-clipboard": "^3.3.3",
+        "d3-array": "^3.2.4",
+        "d3-axis": "^3.0.0",
+        "d3-dsv": "^3.0.1",
+        "d3-scale": "^4.0.2",
         "d3-scale-chromatic": "^3.1.0",
-        "d3-selection": "^2.0.0",
+        "d3-selection": "^3.0.0",
         "date-fns": "^2.28.0",
-        "echarts": "^5.4.3",
+        "echarts": "^5.5.1",
         "file-saver": "^2.0.5",
         "hjson": "^3.2.2",
         "json-bigint-native": "^1.2.0",
         "lodash.debounce": "^4.0.8",
         "lodash.escape": "^4.0.1",
-        "memoize-one": "^5.1.1",
+        "memoize-one": "^5.2.1",
         "numeral": "^2.0.6",
-        "path-parse": "^1.0.7",
-        "react": "^18.2.0",
+        "react": "^18.3.1",
         "react-ace": "^10.1.0",
         "react-diff-viewer": "^3.1.1",
-        "react-dom": "^18.2.0",
+        "react-dom": "^18.3.1",
         "react-router": "^5.3.4",
         "react-router-dom": "^5.3.4",
         "react-splitter-layout": "^4.0.0",
         "react-table": "~6.11.5",
         "regenerator-runtime": "^0.13.7",
-        "tslib": "^2.3.1",
-        "uuid": "^7.0.2",
-        "zustand": "^4.3.2"
+        "tslib": "^2.8.0",
+        "uuid": "^7.0.3",
+        "zustand": "^4.5.5"
       },
       "devDependencies": {
         "@awesome-code-style/eslint-config": "^6.0.0",
         "@awesome-code-style/prettier-config": "^4.0.0",
         "@awesome-code-style/stylelint-config": "^6.0.0",
-        "@testing-library/react": "^14.0.0",
+        "@testing-library/dom": "^10.4.0",
+        "@testing-library/react": "^16.0.1",
         "@types/classnames": "^2.2.9",
-        "@types/d3-array": "^2.12.3",
-        "@types/d3-axis": "^2.1.3",
-        "@types/d3-dsv": "^2.0.0",
-        "@types/d3-scale": "^3.3.2",
+        "@types/d3-array": "^3.2.1",
+        "@types/d3-axis": "^3.0.6",
+        "@types/d3-dsv": "^3.0.7",
+        "@types/d3-scale": "^4.0.8",
         "@types/d3-scale-chromatic": "^3.0.3",
-        "@types/d3-selection": "^2.0.1",
-        "@types/enzyme": "^3.10.17",
+        "@types/d3-selection": "^3.0.11",
+        "@types/enzyme": "^3.10.18",
         "@types/enzyme-adapter-react-16": "^1.0.9",
         "@types/file-saver": "^2.0.7",
         "@types/hjson": "^2.4.6",
         "@types/jest": "^27.4.0",
-        "@types/lodash.debounce": "^4.0.6",
-        "@types/lodash.escape": "^4.0.6",
+        "@types/lodash.debounce": "^4.0.8",
+        "@types/lodash.escape": "^4.0.9",
         "@types/memoize-one": "^4.1.1",
-        "@types/node": "^12.11.7",
-        "@types/numeral": "^0.0.26",
-        "@types/react": "^18.2.42",
-        "@types/react-dom": "^18.2.17",
+        "@types/node": "^18.19.56",
+        "@types/numeral": "^2.0.5",
+        "@types/react": "^18.3.11",
+        "@types/react-dom": "^18.3.1",
         "@types/react-router-dom": "^5.3.3",
         "@types/react-splitter-layout": "^3.0.5",
         "@types/react-table": "6.8.5",
@@ -105,10 +104,10 @@
         "snarkdown": "^2.0.0",
         "style-loader": "^2.0.0",
         "stylelint": "^16.1.0",
-        "ts-jest": "^29.1.2",
-        "ts-loader": "^9.4.2",
-        "ts-node": "^10.9.1",
-        "typescript": "^4.9.5",
+        "ts-jest": "^29.2.5",
+        "ts-loader": "^9.5.1",
+        "ts-node": "^10.9.2",
+        "typescript": "~5.5.4",
         "webpack": "^5.89.0",
         "webpack-bundle-analyzer": "^4.10.1",
         "webpack-cli": "^5.1.4",
@@ -787,20 +786,28 @@
       "dev": true
     },
     "node_modules/@blueprintjs/colors": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/colors/-/colors-5.1.1.tgz",
-      "integrity": "sha512-Mni/GgCYLaMf5U5zsCN42skOG49w3U0QmUFuJgFZ/1pv+3cHF/9xR4L4dXcj5DtgJoHkmUbK36PR5mdFB65WEA==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/colors/-/colors-5.1.2.tgz",
+      "integrity": "sha512-7CWwVsXK4YTN9Z/wkjnS3p7VE8YfIXXv2UaySAbtcw6rBkmoSHjLRtfohSA5yNy8xYTQ4KY2odKZSUW0W/Nltw==",
+      "license": "Apache-2.0",
       "dependencies": {
         "tslib": "~2.6.2"
       }
     },
+    "node_modules/@blueprintjs/colors/node_modules/tslib": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+      "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
+      "license": "0BSD"
+    },
     "node_modules/@blueprintjs/core": {
-      "version": "5.10.5",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-5.10.5.tgz",
-      "integrity": "sha512-mvTB+Ox31nt9830hEgVOi/A53QI71hpOzV/7i3ZgzzrYAaEGgFMcwa7X0kn9PSOGfxw87nWL9VpYSirnzpC6lw==",
+      "version": "5.13.1",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-5.13.1.tgz",
+      "integrity": "sha512-PDZ9X/xGBetwU2AqQuCDGVWZZMmt6+/BCnmoKXxsBBZIuWQuHDiKcwvb9rzup0htsA6P7KGl5aw7ocmDvZPpBw==",
+      "license": "Apache-2.0",
       "dependencies": {
-        "@blueprintjs/colors": "^5.1.1",
-        "@blueprintjs/icons": "^5.10.0",
+        "@blueprintjs/colors": "^5.1.2",
+        "@blueprintjs/icons": "^5.13.0",
         "@popperjs/core": "^2.11.8",
         "classnames": "^2.3.1",
         "normalize.css": "^8.0.1",
@@ -825,14 +832,21 @@
         }
       }
     },
+    "node_modules/@blueprintjs/core/node_modules/tslib": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+      "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
+      "license": "0BSD"
+    },
     "node_modules/@blueprintjs/datetime": {
-      "version": "5.3.7",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/datetime/-/datetime-5.3.7.tgz",
-      "integrity": "sha512-6rDMqSU8jeiAlMxS30MYAQc4X1sfMGWwFjfs3IoKxJfcfieN+ytus5G+if6zjsSKUjdhL9+qBTgKRo3VJJEQ3g==",
+      "version": "5.3.11",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/datetime/-/datetime-5.3.11.tgz",
+      "integrity": "sha512-6MMictO0OPc6gRoPt/gZ6V5tFTKj9d2M3Mpn4alNa74uZF/Y1FjBfOvSPOsV7JajBOVjqiaSEBRZIFHrhRr91Q==",
+      "license": "Apache-2.0",
       "dependencies": {
-        "@blueprintjs/core": "^5.10.5",
-        "@blueprintjs/icons": "^5.10.0",
-        "@blueprintjs/select": "^5.2.1",
+        "@blueprintjs/core": "^5.13.1",
+        "@blueprintjs/icons": "^5.13.0",
+        "@blueprintjs/select": "^5.2.5",
         "classnames": "^2.3.1",
         "date-fns": "^2.28.0",
         "date-fns-tz": "^2.0.0",
@@ -850,14 +864,21 @@
         }
       }
     },
+    "node_modules/@blueprintjs/datetime/node_modules/tslib": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+      "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
+      "license": "0BSD"
+    },
     "node_modules/@blueprintjs/datetime2": {
-      "version": "2.3.7",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/datetime2/-/datetime2-2.3.7.tgz",
-      "integrity": "sha512-Q6IBo5hcvoDtH9FsFaysZhadXNS6iyhuyaRY+rJD3y3qLaUt6wiYDhw5LN6L4wuasbPO1g0lVDwY6bkGuIG4jg==",
+      "version": "2.3.11",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/datetime2/-/datetime2-2.3.11.tgz",
+      "integrity": "sha512-o9NYfUAb8xYiGZQ56tn6CXm8jDNxwSxJvZgrzs6EVYCuCFrsBr9WAjy0p768ZbZlzuWwClbEjslr8s+E6etU/w==",
+      "license": "Apache-2.0",
       "dependencies": {
-        "@blueprintjs/core": "^5.10.5",
-        "@blueprintjs/datetime": "^5.3.7",
-        "@blueprintjs/icons": "^5.10.0",
+        "@blueprintjs/core": "^5.13.1",
+        "@blueprintjs/datetime": "^5.3.11",
+        "@blueprintjs/icons": "^5.13.0",
         "classnames": "^2.3.1",
         "date-fns": "^2.28.0",
         "react-day-picker": "^8.10.0",
@@ -888,10 +909,17 @@
         "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       }
     },
+    "node_modules/@blueprintjs/datetime2/node_modules/tslib": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+      "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
+      "license": "0BSD"
+    },
     "node_modules/@blueprintjs/icons": {
-      "version": "5.10.0",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-5.10.0.tgz",
-      "integrity": "sha512-Qp9H2U2/m6+p0Lh9ACuF2O6o0uVINMFimo+EgasgGaLx8SkJsl7ApB3hYQ+iRSYlQWotnoqd4Vtzj4c7sKukHQ==",
+      "version": "5.13.0",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-5.13.0.tgz",
+      "integrity": "sha512-L096dBjzfnWW7fWXM311S2C/5Zn0EuEK9q6G84QvWP0BZJOTowU1EIWLj90IgGtNajld/3ZUAj6eJf+ryt/kjQ==",
+      "license": "Apache-2.0",
       "dependencies": {
         "change-case": "^4.1.2",
         "classnames": "^2.3.1",
@@ -908,13 +936,20 @@
         }
       }
     },
+    "node_modules/@blueprintjs/icons/node_modules/tslib": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+      "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
+      "license": "0BSD"
+    },
     "node_modules/@blueprintjs/select": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/select/-/select-5.2.1.tgz",
-      "integrity": "sha512-rXM9ik2mLd394g+sxwbHv2bGfrLbcdY9gFftqmJmm3vu7XJ2kRrsC1oOlNSC4dNBpUdHnyAzBTHEu81B1LkRbQ==",
+      "version": "5.2.5",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/select/-/select-5.2.5.tgz",
+      "integrity": "sha512-mO9r5iQ4uxdgScTyNSNSLq3DUZgFtI3dAO1VRUNONBGPcSROVi+3ixYYynwcad18ftAZz/WuUD9ZHqVyDwXmbQ==",
+      "license": "Apache-2.0",
       "dependencies": {
-        "@blueprintjs/core": "^5.10.5",
-        "@blueprintjs/icons": "^5.10.0",
+        "@blueprintjs/core": "^5.13.1",
+        "@blueprintjs/icons": "^5.13.0",
         "classnames": "^2.3.1",
         "tslib": "~2.6.2"
       },
@@ -929,6 +964,12 @@
         }
       }
     },
+    "node_modules/@blueprintjs/select/node_modules/tslib": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+      "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==",
+      "license": "0BSD"
+    },
     "node_modules/@cspotcode/source-map-support": {
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -2047,9 +2088,10 @@
       }
     },
     "node_modules/@druid-toolkit/query": {
-      "version": "0.22.22",
-      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.22.tgz",
-      "integrity": "sha512-QCzIFr0H6uk5KsGEKO/Vk1fWWurf54ENagFCygsKVk3p2sCRmpZ7WGKeQ5i3AFh2847tOeopZI0OniEv2yTLGQ==",
+      "version": "0.22.23",
+      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.23.tgz",
+      "integrity": "sha512-yQOUAQJP63rzsTCdLcqNB8aRtsYPw8rYBfPSXc4zfAA4y/GJc9OJeHcLFRMdUtpwBtm0ueARMUlTSQcTsyV8gQ==",
+      "license": "Apache-2.0",
       "dependencies": {
         "tslib": "^2.5.2"
       }
@@ -2346,9 +2388,10 @@
       }
     },
     "node_modules/@fontsource/open-sans": {
-      "version": "5.0.28",
-      "resolved": "https://registry.npmjs.org/@fontsource/open-sans/-/open-sans-5.0.28.tgz",
-      "integrity": "sha512-hBvJHY76pJT/JynGUB5EXWhnzjYfLdcMn655J5p1v9lTT9HdQSy+keq2KPVXO2Htlg998BBa3p6u/jlrZ6w0kg=="
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/@fontsource/open-sans/-/open-sans-5.1.0.tgz",
+      "integrity": "sha512-g+mjF8gWUDwck9DrRCkhmFeEj7fskjtKZJKAQguVzSg93lc6ThakTHMRgs0dZfe5qBbktrV839tDrb4bIDyZSA==",
+      "license": "OFL-1.1"
     },
     "node_modules/@humanfs/core": {
       "version": "0.19.0",
@@ -3433,22 +3476,23 @@
       }
     },
     "node_modules/@testing-library/dom": {
-      "version": "9.3.0",
-      "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-9.3.0.tgz",
-      "integrity": "sha512-Dffe68pGwI6WlLRYR2I0piIkyole9cSBH5jGQKCGMRpHW5RHCqAUaqc2Kv0tUyd4dU4DLPKhJIjyKOnjv4tuUw==",
+      "version": "10.4.0",
+      "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz",
+      "integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "@babel/code-frame": "^7.10.4",
         "@babel/runtime": "^7.12.5",
         "@types/aria-query": "^5.0.1",
-        "aria-query": "^5.0.0",
+        "aria-query": "5.3.0",
         "chalk": "^4.1.0",
         "dom-accessibility-api": "^0.5.9",
         "lz-string": "^1.5.0",
         "pretty-format": "^27.0.2"
       },
       "engines": {
-        "node": ">=14"
+        "node": ">=18"
       }
     },
     "node_modules/@testing-library/dom/node_modules/ansi-styles": {
@@ -3456,6 +3500,7 @@
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
       "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "color-convert": "^2.0.1"
       },
@@ -3471,6 +3516,7 @@
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
       "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "ansi-styles": "^4.1.0",
         "supports-color": "^7.1.0"
@@ -3487,6 +3533,7 @@
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
       "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "color-name": "~1.1.4"
       },
@@ -3498,13 +3545,15 @@
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
-      "dev": true
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/@testing-library/dom/node_modules/has-flag": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
       "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
       "dev": true,
+      "license": "MIT",
       "engines": {
         "node": ">=8"
       }
@@ -3514,6 +3563,7 @@
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
       "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "has-flag": "^4.0.0"
       },
@@ -3522,21 +3572,31 @@
       }
     },
     "node_modules/@testing-library/react": {
-      "version": "14.0.0",
-      "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-14.0.0.tgz",
-      "integrity": "sha512-S04gSNJbYE30TlIMLTzv6QCTzt9AqIF5y6s6SzVFILNcNvbV/jU96GeiTPillGQo+Ny64M/5PV7klNYYgv5Dfg==",
+      "version": "16.0.1",
+      "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.0.1.tgz",
+      "integrity": "sha512-dSmwJVtJXmku+iocRhWOUFbrERC76TX2Mnf0ATODz8brzAZrMBbzLwQixlBSanZxR6LddK3eiwpSFZgDET1URg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@babel/runtime": "^7.12.5",
-        "@testing-library/dom": "^9.0.0",
-        "@types/react-dom": "^18.0.0"
+        "@babel/runtime": "^7.12.5"
       },
       "engines": {
-        "node": ">=14"
+        "node": ">=18"
       },
       "peerDependencies": {
+        "@testing-library/dom": "^10.0.0",
+        "@types/react": "^18.0.0",
+        "@types/react-dom": "^18.0.0",
         "react": "^18.0.0",
         "react-dom": "^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
       }
     },
     "node_modules/@tootallnate/once": {
@@ -3573,10 +3633,11 @@
       "dev": true
     },
     "node_modules/@types/aria-query": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.1.tgz",
-      "integrity": "sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==",
-      "dev": true
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz",
+      "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/@types/babel__core": {
       "version": "7.20.5",
@@ -3673,33 +3734,37 @@
       }
     },
     "node_modules/@types/d3-array": {
-      "version": "2.12.3",
-      "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-2.12.3.tgz",
-      "integrity": "sha512-hN879HLPTVqZV3FQEXy7ptt083UXwguNbnxdTGzVW4y4KjX5uyNKljrQixZcSJfLyFirbpUokxpXtvR+N5+KIg==",
-      "dev": true
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz",
+      "integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/@types/d3-axis": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-2.1.3.tgz",
-      "integrity": "sha512-QjXjwZ0xzyrW2ndkmkb09ErgWDEYtbLBKGui73QLMFm3woqWpxptfD5Y7vqQdybMcu7WEbjZ5q+w2w5+uh2IjA==",
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz",
+      "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@types/d3-selection": "^2"
+        "@types/d3-selection": "*"
       }
     },
     "node_modules/@types/d3-dsv": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-2.0.3.tgz",
-      "integrity": "sha512-15sp4Z+ZVWuZuV0QEDu4cu/0C5vlD+JYXaUMDs8JTWpTJjcrAtjyR1vVwEfbgmU5kLNOOMRTlDCYyWWFx7eh/w==",
-      "dev": true
+      "version": "3.0.7",
+      "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz",
+      "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/@types/d3-scale": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-3.3.2.tgz",
-      "integrity": "sha512-gGqr7x1ost9px3FvIfUMi5XA/F/yAf4UkUDtdQhpH92XCT0Oa7zkkRzY61gPVJq+DxpHn/btouw5ohWkbBsCzQ==",
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.8.tgz",
+      "integrity": "sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "@types/d3-time": "^2"
+        "@types/d3-time": "*"
       }
     },
     "node_modules/@types/d3-scale-chromatic": {
@@ -3709,10 +3774,11 @@
       "dev": true
     },
     "node_modules/@types/d3-selection": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-2.0.1.tgz",
-      "integrity": "sha512-3mhtPnGE+c71rl/T5HMy+ykg7migAZ4T6gzU0HxpgBFKcasBrSnwRbYV1/UZR6o5fkpySxhWxAhd7yhjj8jL7g==",
-      "dev": true
+      "version": "3.0.11",
+      "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz",
+      "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/@types/d3-time": {
       "version": "2.1.1",
@@ -3721,10 +3787,11 @@
       "dev": true
     },
     "node_modules/@types/enzyme": {
-      "version": "3.10.17",
-      "resolved": "https://registry.npmjs.org/@types/enzyme/-/enzyme-3.10.17.tgz",
-      "integrity": "sha512-4+CMYPx8cToXnF1WxUdjKCZFSrslJqkIW5N5D27stzJsnJmw4dWbOcmhPERJpe+CufV0LanV7BgTWuj63Z2/Vg==",
+      "version": "3.10.18",
+      "resolved": "https://registry.npmjs.org/@types/enzyme/-/enzyme-3.10.18.tgz",
+      "integrity": "sha512-RaO/TyyHZvXkpzinbMTZmd/S5biU4zxkvDsn22ujC29t9FMSzq8tnn8f2MxQ2P8GVhFRG5jTAL05DXKyTtpEQQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "@types/cheerio": "*",
         "@types/react": "^16"
@@ -3877,19 +3944,21 @@
       "dev": true
     },
     "node_modules/@types/lodash.debounce": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/@types/lodash.debounce/-/lodash.debounce-4.0.6.tgz",
-      "integrity": "sha512-4WTmnnhCfDvvuLMaF3KV4Qfki93KebocUF45msxhYyjMttZDQYzHkO639ohhk8+oco2cluAFL3t5+Jn4mleylQ==",
+      "version": "4.0.9",
+      "resolved": "https://registry.npmjs.org/@types/lodash.debounce/-/lodash.debounce-4.0.9.tgz",
+      "integrity": "sha512-Ma5JcgTREwpLRwMM+XwBR7DaWe96nC38uCBDFKZWbNKD+osjVzdpnUSwBcqCptrp16sSOLBAUb50Car5I0TCsQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "@types/lodash": "*"
       }
     },
     "node_modules/@types/lodash.escape": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/@types/lodash.escape/-/lodash.escape-4.0.6.tgz",
-      "integrity": "sha512-/CI97B3wf1R4oD4yotsGoX/5bobL/RC8olTQ16iunzdjufcdD8GyIvNDatg1IfzVKaFBZmxuY0+WQDpdaQHLzg==",
+      "version": "4.0.9",
+      "resolved": "https://registry.npmjs.org/@types/lodash.escape/-/lodash.escape-4.0.9.tgz",
+      "integrity": "sha512-YR9R6RU9D36OWrhVZ+r4fY8mMXQtofnLXAcE6CkKIxdH66uiCdEjkhxx/eiJnYpxjOFmPbWBUUvdv8lckO823Q==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "@types/lodash": "*"
       }
@@ -3907,10 +3976,14 @@
       "dev": true
     },
     "node_modules/@types/node": {
-      "version": "12.11.7",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.11.7.tgz",
-      "integrity": "sha512-JNbGaHFCLwgHn/iCckiGSOZ1XYHsKFwREtzPwSGCVld1SGhOlmZw2D4ZI94HQCrBHbADzW9m4LER/8olJTRGHA==",
-      "dev": true
+      "version": "18.19.56",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.56.tgz",
+      "integrity": "sha512-4EMJlWwwGnVPflJAtM14p9eVSa6BOv5b92mCsh5zcM1UagNtEtrbbtaE6WE1tw2TabavatnwqXjlIpcAEuJJNg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~5.26.4"
+      }
     },
     "node_modules/@types/node-forge": {
       "version": "1.3.10",
@@ -3922,10 +3995,11 @@
       }
     },
     "node_modules/@types/numeral": {
-      "version": "0.0.26",
-      "resolved": "https://registry.npmjs.org/@types/numeral/-/numeral-0.0.26.tgz",
-      "integrity": "sha512-DwCsRqeOWopdEsm5KLTxKVKDSDoj+pzZD1vlwu1GQJ6IF3RhjuleYlRwyRH6MJLGaf3v8wFTnC6wo3yYfz0bnA==",
-      "dev": true
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@types/numeral/-/numeral-2.0.5.tgz",
+      "integrity": "sha512-kH8I7OSSwQu9DS9JYdFWbuvhVzvFRoCPCkGxNwoGgaPeDfEPJlcxNvEOypZhQ3XXHsGbfIuYcxcJxKUfJHnRfw==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/@types/parse-json": {
       "version": "4.0.0",
@@ -3950,20 +4024,21 @@
       "dev": true
     },
     "node_modules/@types/react": {
-      "version": "18.2.42",
-      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.42.tgz",
-      "integrity": "sha512-c1zEr96MjakLYus/wPnuWDo1/zErfdU9rNsIGmE+NV71nx88FG9Ttgo5dqorXTu/LImX2f63WBP986gJkMPNbA==",
+      "version": "18.3.11",
+      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.11.tgz",
+      "integrity": "sha512-r6QZ069rFTjrEYgFdOck1gK7FLVsgJE7tTz0pQBczlBNUhBNk0MQH4UbnFSwjpQLMkLzgqvBBa+qGpLje16eTQ==",
+      "license": "MIT",
       "dependencies": {
         "@types/prop-types": "*",
-        "@types/scheduler": "*",
         "csstype": "^3.0.2"
       }
     },
     "node_modules/@types/react-dom": {
-      "version": "18.2.17",
-      "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.17.tgz",
-      "integrity": "sha512-rvrT/M7Df5eykWFxn6MYt5Pem/Dbyc1N8Y0S9Mrkw2WFCRiqUgw9P7ul2NpwsXCSM1DVdENzdG9J5SreqfAIWg==",
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.1.tgz",
+      "integrity": "sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "@types/react": "*"
       }
@@ -4012,11 +4087,6 @@
       "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",
       "dev": true
     },
-    "node_modules/@types/scheduler": {
-      "version": "0.16.8",
-      "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz",
-      "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A=="
-    },
     "node_modules/@types/send": {
       "version": "0.17.4",
       "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
@@ -4631,9 +4701,10 @@
       }
     },
     "node_modules/ace-builds": {
-      "version": "1.4.14",
-      "resolved": "https://registry.npmjs.org/ace-builds/-/ace-builds-1.4.14.tgz",
-      "integrity": "sha512-NBOQlm9+7RBqRqZwimpgquaLeTJFayqb9UEPtTkpC3TkkwDnlsT/TwsCC0svjt9kEZ6G9mH5AEOHSz6Q/HrzQQ=="
+      "version": "1.5.3",
+      "resolved": "https://registry.npmjs.org/ace-builds/-/ace-builds-1.5.3.tgz",
+      "integrity": "sha512-WN5BKR2aTSuBmisO8jo3Fytk6sOmJGki82v/Boeic81IgYN8pFHNkXq2anDF0XkmfDWMqLbRoW9sjc/GtKzQbQ==",
+      "license": "BSD-3-Clause"
     },
     "node_modules/acorn": {
       "version": "8.13.0",
@@ -4881,49 +4952,15 @@
       }
     },
     "node_modules/aria-query": {
-      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.1.3.tgz",
-      "integrity": "sha512-R5iJ5lkuHybztUfuOAznmboyjWq8O6sqNqtK7CLOqdydi54VNbORp49mb14KbWgG1QD3JFO9hJdZ+y4KutfdOQ==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz",
+      "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==",
       "dev": true,
+      "license": "Apache-2.0",
       "dependencies": {
-        "deep-equal": "^2.0.5"
+        "dequal": "^2.0.3"
       }
     },
-    "node_modules/aria-query/node_modules/deep-equal": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz",
-      "integrity": "sha512-lKdkdV6EOGoVn65XaOsPdH4rMxTZOnmFyuIkMjM1i5HHCbfjC97dawgTAy0deYNfuqUqW+Q5VrVaQYtUpSd6yQ==",
-      "dev": true,
-      "dependencies": {
-        "array-buffer-byte-length": "^1.0.0",
-        "call-bind": "^1.0.2",
-        "es-get-iterator": "^1.1.3",
-        "get-intrinsic": "^1.2.0",
-        "is-arguments": "^1.1.1",
-        "is-array-buffer": "^3.0.2",
-        "is-date-object": "^1.0.5",
-        "is-regex": "^1.1.4",
-        "is-shared-array-buffer": "^1.0.2",
-        "isarray": "^2.0.5",
-        "object-is": "^1.1.5",
-        "object-keys": "^1.1.1",
-        "object.assign": "^4.1.4",
-        "regexp.prototype.flags": "^1.5.0",
-        "side-channel": "^1.0.4",
-        "which-boxed-primitive": "^1.0.2",
-        "which-collection": "^1.0.1",
-        "which-typed-array": "^1.1.9"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/aria-query/node_modules/isarray": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz",
-      "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==",
-      "dev": true
-    },
     "node_modules/array-buffer-byte-length": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz",
@@ -5123,6 +5160,13 @@
         "node": ">=8"
       }
     },
+    "node_modules/async": {
+      "version": "3.2.6",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz",
+      "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -5188,9 +5232,10 @@
       }
     },
     "node_modules/axios": {
-      "version": "1.7.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz",
-      "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==",
+      "version": "1.7.7",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz",
+      "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==",
+      "license": "MIT",
       "dependencies": {
         "follow-redirects": "^1.15.6",
         "form-data": "^4.0.0",
@@ -5644,6 +5689,7 @@
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-4.1.2.tgz",
       "integrity": "sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==",
+      "license": "MIT",
       "dependencies": {
         "pascal-case": "^3.1.2",
         "tslib": "^2.0.3"
@@ -5682,6 +5728,7 @@
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/capital-case/-/capital-case-1.0.4.tgz",
       "integrity": "sha512-ds37W8CytHgwnhGGTi88pcPyR15qoNkOpYwmMMfnWqqWgESapLqvDx6huFjQ5vqWSn2Z06173XNA7LtMOeUh1A==",
+      "license": "MIT",
       "dependencies": {
         "no-case": "^3.0.4",
         "tslib": "^2.0.3",
@@ -5705,6 +5752,7 @@
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/change-case/-/change-case-4.1.2.tgz",
       "integrity": "sha512-bSxY2ws9OtviILG1EiY5K7NNxkqg/JnRnFxLtKQ96JaviiIxi7djMrSd0ECT9AC+lttClmYwKw53BWpOMblo7A==",
+      "license": "MIT",
       "dependencies": {
         "camel-case": "^4.1.2",
         "capital-case": "^1.0.4",
@@ -5895,7 +5943,8 @@
     "node_modules/commander": {
       "version": "2.20.0",
       "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
-      "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ=="
+      "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==",
+      "dev": true
     },
     "node_modules/compressible": {
       "version": "2.0.18",
@@ -5955,6 +6004,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/constant-case/-/constant-case-3.0.4.tgz",
       "integrity": "sha512-I2hSBi7Vvs7BEuJDr5dDHfzb/Ruj3FyvFyh7KLilAjNQw3Be+xgqUBA2W6scVEcL0hL1dwPRtIqEPVUCKkSsyQ==",
+      "license": "MIT",
       "dependencies": {
         "no-case": "^3.0.4",
         "tslib": "^2.0.3",
@@ -6027,24 +6077,14 @@
       "dev": true
     },
     "node_modules/copy-to-clipboard": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.2.0.tgz",
-      "integrity": "sha512-eOZERzvCmxS8HWzugj4Uxl8OJxa7T2k1Gi0X5qavwydHIfuSHq2dTD09LOg/XyGq4Zpb5IsR/2OJ5lbOegz78w==",
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz",
+      "integrity": "sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==",
+      "license": "MIT",
       "dependencies": {
         "toggle-selection": "^1.0.6"
       }
     },
-    "node_modules/core-js": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.10.1.tgz",
-      "integrity": "sha512-pwCxEXnj27XG47mu7SXAwhLP3L5CrlvCB91ANUkIz40P27kUcvNfSdvyZJ9CLHiVoKSp+TTChMQMSKQEH/IQxA==",
-      "deprecated": "core-js@<3.23.3 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Some versions have web compatibility issues. Please, upgrade your dependencies to the actual version of core-js.",
-      "hasInstallScript": true,
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/core-js"
-      }
-    },
     "node_modules/core-util-is": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
@@ -6447,42 +6487,79 @@
       "integrity": "sha512-KxnUB0ZMlnUWCsx2Z8MUsr6qV6ja1w9ArPErJaJaF8a5SOWoHLIszeCTKGRGRgtLgYrs1E8CHkNSP1VZTTPc9g=="
     },
     "node_modules/d3-array": {
-      "version": "2.12.1",
-      "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz",
-      "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==",
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
+      "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
+      "license": "ISC",
       "dependencies": {
-        "internmap": "^1.0.0"
+        "internmap": "1 - 2"
+      },
+      "engines": {
+        "node": ">=12"
       }
     },
     "node_modules/d3-axis": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-2.1.0.tgz",
-      "integrity": "sha512-z/G2TQMyuf0X3qP+Mh+2PimoJD41VOCjViJzT0BHeL/+JQAofkiWZbWxlwFGb1N8EN+Cl/CW+MUKbVzr1689Cw=="
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz",
+      "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
     },
     "node_modules/d3-color": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-2.0.0.tgz",
-      "integrity": "sha512-SPXi0TSKPD4g9tw0NMZFnR95XVgUZiBH+uUTqQuDu1OsE2zomHU7ho0FISciaPvosimixwHFl3WHLGabv6dDgQ=="
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
+      "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
     },
     "node_modules/d3-dsv": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-2.0.0.tgz",
-      "integrity": "sha512-E+Pn8UJYx9mViuIUkoc93gJGGYut6mSDKy2+XaPwccwkRGlR+LO97L2VCCRjQivTwLHkSnAJG7yo00BWY6QM+w==",
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz",
+      "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==",
+      "license": "ISC",
       "dependencies": {
-        "commander": "2",
-        "iconv-lite": "0.4",
+        "commander": "7",
+        "iconv-lite": "0.6",
         "rw": "1"
       },
       "bin": {
-        "csv2json": "bin/dsv2json",
-        "csv2tsv": "bin/dsv2dsv",
-        "dsv2dsv": "bin/dsv2dsv",
-        "dsv2json": "bin/dsv2json",
-        "json2csv": "bin/json2dsv",
-        "json2dsv": "bin/json2dsv",
-        "json2tsv": "bin/json2dsv",
-        "tsv2csv": "bin/dsv2dsv",
-        "tsv2json": "bin/dsv2json"
+        "csv2json": "bin/dsv2json.js",
+        "csv2tsv": "bin/dsv2dsv.js",
+        "dsv2dsv": "bin/dsv2dsv.js",
+        "dsv2json": "bin/dsv2json.js",
+        "json2csv": "bin/json2dsv.js",
+        "json2dsv": "bin/json2dsv.js",
+        "json2tsv": "bin/json2dsv.js",
+        "tsv2csv": "bin/dsv2dsv.js",
+        "tsv2json": "bin/dsv2json.js"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/d3-dsv/node_modules/commander": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz",
+      "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 10"
+      }
+    },
+    "node_modules/d3-dsv/node_modules/iconv-lite": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+      "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
       }
     },
     "node_modules/d3-format": {
@@ -6491,23 +6568,31 @@
       "integrity": "sha512-TUswGe6hfguUX1CtKxyG2nymO+1lyThbkS1ifLX0Sr+dOQtAD5gkrffpHnx+yHNKUZ0Bmg5T4AjUQwugPDrm0g=="
     },
     "node_modules/d3-interpolate": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-2.0.1.tgz",
-      "integrity": "sha512-c5UhwwTs/yybcmTpAVqwSFl6vrQ8JZJoT5F7xNFK9pymv5C0Ymcc9/LIJHtYIggg/yS9YHw8i8O8tgb9pupjeQ==",
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
+      "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
+      "license": "ISC",
       "dependencies": {
-        "d3-color": "1 - 2"
+        "d3-color": "1 - 3"
+      },
+      "engines": {
+        "node": ">=12"
       }
     },
     "node_modules/d3-scale": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-3.3.0.tgz",
-      "integrity": "sha512-1JGp44NQCt5d1g+Yy+GeOnZP7xHo0ii8zsQp6PGzd+C1/dl0KGsp9A7Mxwp+1D1o4unbTTxVdU/ZOIEBoeZPbQ==",
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
+      "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
+      "license": "ISC",
       "dependencies": {
-        "d3-array": "^2.3.0",
-        "d3-format": "1 - 2",
-        "d3-interpolate": "1.2.0 - 2",
-        "d3-time": "^2.1.1",
-        "d3-time-format": "2 - 3"
+        "d3-array": "2.10.0 - 3",
+        "d3-format": "1 - 3",
+        "d3-interpolate": "1.2.0 - 3",
+        "d3-time": "2.1.1 - 3",
+        "d3-time-format": "2 - 4"
+      },
+      "engines": {
+        "node": ">=12"
       }
     },
     "node_modules/d3-scale-chromatic": {
@@ -6522,6 +6607,15 @@
         "node": ">=12"
       }
     },
+    "node_modules/d3-scale/node_modules/d3-array": {
+      "version": "2.12.1",
+      "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz",
+      "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "internmap": "^1.0.0"
+      }
+    },
     "node_modules/d3-scale/node_modules/d3-time": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-2.1.1.tgz",
@@ -6531,9 +6625,13 @@
       }
     },
     "node_modules/d3-selection": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-2.0.0.tgz",
-      "integrity": "sha512-XoGGqhLUN/W14NmaqcO/bb1nqjDAw5WtSYb2X8wiuQWvSZUsUVYsOSkOybUrNvcBjaywBdYPy03eXHMXjk9nZA=="
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz",
+      "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==",
+      "license": "ISC",
+      "engines": {
+        "node": ">=12"
+      }
     },
     "node_modules/d3-time": {
       "version": "1.1.0",
@@ -6789,6 +6887,16 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/dequal": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
+      "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/destroy": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
@@ -6881,7 +6989,8 @@
       "version": "0.5.16",
       "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz",
       "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==",
-      "dev": true
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/dom-helpers": {
       "version": "5.2.1",
@@ -6909,6 +7018,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.4.tgz",
       "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==",
+      "license": "MIT",
       "dependencies": {
         "no-case": "^3.0.4",
         "tslib": "^2.0.3"
@@ -6921,12 +7031,13 @@
       "dev": true
     },
     "node_modules/echarts": {
-      "version": "5.4.3",
-      "resolved": "https://registry.npmjs.org/echarts/-/echarts-5.4.3.tgz",
-      "integrity": "sha512-mYKxLxhzy6zyTi/FaEbJMOZU1ULGEQHaeIeuMR5L+JnJTpz+YR03mnnpBhbR4+UYJAgiXgpyTVLffPAjOTLkZA==",
+      "version": "5.5.1",
+      "resolved": "https://registry.npmjs.org/echarts/-/echarts-5.5.1.tgz",
+      "integrity": "sha512-Fce8upazaAXUVUVsjgV6mBnGuqgO+JNDlcgF79Dksy4+wgGpQB2lmYoO4TSweFg/mZITdpGHomw/cNBJZj1icA==",
+      "license": "Apache-2.0",
       "dependencies": {
         "tslib": "2.3.0",
-        "zrender": "5.4.4"
+        "zrender": "5.6.0"
       }
     },
     "node_modules/echarts/node_modules/tslib": {
@@ -6940,6 +7051,22 @@
       "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
       "dev": true
     },
+    "node_modules/ejs": {
+      "version": "3.1.10",
+      "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz",
+      "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "jake": "^10.8.5"
+      },
+      "bin": {
+        "ejs": "bin/cli.js"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/electron-to-chromium": {
       "version": "1.5.33",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.33.tgz",
@@ -7116,32 +7243,6 @@
         "node": ">= 0.4"
       }
     },
-    "node_modules/es-get-iterator": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz",
-      "integrity": "sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==",
-      "dev": true,
-      "dependencies": {
-        "call-bind": "^1.0.2",
-        "get-intrinsic": "^1.1.3",
-        "has-symbols": "^1.0.3",
-        "is-arguments": "^1.1.1",
-        "is-map": "^2.0.2",
-        "is-set": "^2.0.2",
-        "is-string": "^1.0.7",
-        "isarray": "^2.0.5",
-        "stop-iteration-iterator": "^1.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
-    "node_modules/es-get-iterator/node_modules/isarray": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz",
-      "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==",
-      "dev": true
-    },
     "node_modules/es-iterator-helpers": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.1.0.tgz",
@@ -8288,6 +8389,39 @@
       "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.5.tgz",
       "integrity": "sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA=="
     },
+    "node_modules/filelist": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz",
+      "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "minimatch": "^5.0.1"
+      }
+    },
+    "node_modules/filelist/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/filelist/node_modules/minimatch": {
+      "version": "5.1.6",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
+      "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/fill-range": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
@@ -8862,6 +8996,7 @@
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/header-case/-/header-case-2.0.4.tgz",
       "integrity": "sha512-H/vuk5TEEVZwrR0lp2zed9OCo1uAILMlx0JEMgC26rzyJJ3N1v6XkwHHXJQdR2doSjcGPM6OKPYoJgf0plJ11Q==",
+      "license": "MIT",
       "dependencies": {
         "capital-case": "^1.0.4",
         "tslib": "^2.0.3"
@@ -9173,6 +9308,7 @@
       "version": "0.4.24",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
       "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "dev": true,
       "dependencies": {
         "safer-buffer": ">= 2.1.2 < 3"
       },
@@ -9338,22 +9474,6 @@
         "node": ">= 0.10"
       }
     },
-    "node_modules/is-arguments": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz",
-      "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==",
-      "dev": true,
-      "dependencies": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
     "node_modules/is-array-buffer": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz",
@@ -10001,6 +10121,114 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/jake": {
+      "version": "10.9.2",
+      "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz",
+      "integrity": "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "async": "^3.2.3",
+        "chalk": "^4.0.2",
+        "filelist": "^1.0.4",
+        "minimatch": "^3.1.2"
+      },
+      "bin": {
+        "jake": "bin/cli.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/jake/node_modules/ansi-styles": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+      "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "color-convert": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
+    "node_modules/jake/node_modules/chalk": {
+      "version": "4.1.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+      "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/jake/node_modules/color-convert": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+      "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "color-name": "~1.1.4"
+      },
+      "engines": {
+        "node": ">=7.0.0"
+      }
+    },
+    "node_modules/jake/node_modules/color-name": {
+      "version": "1.1.4",
+      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+      "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+      "dev": true,
+      "license": "MIT"
+    },
+    "node_modules/jake/node_modules/has-flag": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+      "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/jake/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/jake/node_modules/supports-color": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+      "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/jest": {
       "version": "29.7.0",
       "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz",
@@ -12443,6 +12671,7 @@
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz",
       "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==",
+      "license": "MIT",
       "dependencies": {
         "tslib": "^2.0.3"
       }
@@ -12464,15 +12693,17 @@
       "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz",
       "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==",
       "dev": true,
+      "license": "MIT",
       "bin": {
         "lz-string": "bin/bin.js"
       }
     },
     "node_modules/make-error": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz",
-      "integrity": "sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==",
-      "dev": true
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
+      "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
+      "dev": true,
+      "license": "ISC"
     },
     "node_modules/makeerror": {
       "version": "1.0.12",
@@ -12522,9 +12753,10 @@
       }
     },
     "node_modules/memoize-one": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.1.1.tgz",
-      "integrity": "sha512-HKeeBpWvqiVJD57ZUAsJNm71eHTykffzcLZVYWiVfQeI1rJtuEaS7hQiEpWfVVk18donPwJEcFKIkCmPJNOhHA=="
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz",
+      "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==",
+      "license": "MIT"
     },
     "node_modules/meow": {
       "version": "13.2.0",
@@ -12758,6 +12990,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/no-case/-/no-case-3.0.4.tgz",
       "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==",
+      "license": "MIT",
       "dependencies": {
         "lower-case": "^2.0.2",
         "tslib": "^2.0.3"
@@ -12920,22 +13153,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/object-is": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz",
-      "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==",
-      "dev": true,
-      "dependencies": {
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.1.3"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/ljharb"
-      }
-    },
     "node_modules/object-keys": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
@@ -13222,6 +13439,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz",
       "integrity": "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==",
+      "license": "MIT",
       "dependencies": {
         "dot-case": "^3.0.4",
         "tslib": "^2.0.3"
@@ -13292,6 +13510,7 @@
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.2.tgz",
       "integrity": "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==",
+      "license": "MIT",
       "dependencies": {
         "no-case": "^3.0.4",
         "tslib": "^2.0.3"
@@ -13301,6 +13520,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/path-case/-/path-case-3.0.4.tgz",
       "integrity": "sha512-qO4qCFjXqVTrcbPt/hQfhTQ+VhFsqNKOPtytgNKkKxSoEp3XPUQ8ObFuePylOIok5gjn69ry8XiULxCwot3Wfg==",
+      "license": "MIT",
       "dependencies": {
         "dot-case": "^3.0.4",
         "tslib": "^2.0.3"
@@ -14584,9 +14804,10 @@
       }
     },
     "node_modules/react": {
-      "version": "18.2.0",
-      "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
-      "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
+      "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==",
+      "license": "MIT",
       "dependencies": {
         "loose-envify": "^1.1.0"
       },
@@ -14642,15 +14863,16 @@
       }
     },
     "node_modules/react-dom": {
-      "version": "18.2.0",
-      "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz",
-      "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==",
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz",
+      "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==",
+      "license": "MIT",
       "dependencies": {
         "loose-envify": "^1.1.0",
-        "scheduler": "^0.23.0"
+        "scheduler": "^0.23.2"
       },
       "peerDependencies": {
-        "react": "^18.2.0"
+        "react": "^18.3.1"
       }
     },
     "node_modules/react-fast-compare": {
@@ -15323,9 +15545,10 @@
       }
     },
     "node_modules/scheduler": {
-      "version": "0.23.0",
-      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz",
-      "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==",
+      "version": "0.23.2",
+      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz",
+      "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==",
+      "license": "MIT",
       "dependencies": {
         "loose-envify": "^1.1.0"
       }
@@ -15452,6 +15675,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/sentence-case/-/sentence-case-3.0.4.tgz",
       "integrity": "sha512-8LS0JInaQMCRoQ7YUytAo/xUu5W2XnQxV2HI/6uM6U7CITS1RqPElr30V6uIqyMKM9lJGRVFy5/4CuzcixNYSg==",
+      "license": "MIT",
       "dependencies": {
         "no-case": "^3.0.4",
         "tslib": "^2.0.3",
@@ -15743,6 +15967,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-3.0.4.tgz",
       "integrity": "sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==",
+      "license": "MIT",
       "dependencies": {
         "dot-case": "^3.0.4",
         "tslib": "^2.0.3"
@@ -15982,18 +16207,6 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/stop-iteration-iterator": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz",
-      "integrity": "sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==",
-      "dev": true,
-      "dependencies": {
-        "internal-slot": "^1.0.4"
-      },
-      "engines": {
-        "node": ">= 0.4"
-      }
-    },
     "node_modules/stream-events": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz",
@@ -16836,28 +17049,31 @@
       }
     },
     "node_modules/ts-jest": {
-      "version": "29.1.2",
-      "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.2.tgz",
-      "integrity": "sha512-br6GJoH/WUX4pu7FbZXuWGKGNDuU7b8Uj77g/Sp7puZV6EXzuByl6JrECvm0MzVzSTkSHWTihsXt+5XYER5b+g==",
+      "version": "29.2.5",
+      "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.5.tgz",
+      "integrity": "sha512-KD8zB2aAZrcKIdGk4OwpJggeLcH1FgrICqDSROWqlnJXGCXK4Mn6FcdK2B6670Xr73lHMG1kHw8R87A0ecZ+vA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "bs-logger": "0.x",
-        "fast-json-stable-stringify": "2.x",
+        "bs-logger": "^0.2.6",
+        "ejs": "^3.1.10",
+        "fast-json-stable-stringify": "^2.1.0",
         "jest-util": "^29.0.0",
         "json5": "^2.2.3",
-        "lodash.memoize": "4.x",
-        "make-error": "1.x",
-        "semver": "^7.5.3",
-        "yargs-parser": "^21.0.1"
+        "lodash.memoize": "^4.1.2",
+        "make-error": "^1.3.6",
+        "semver": "^7.6.3",
+        "yargs-parser": "^21.1.1"
       },
       "bin": {
         "ts-jest": "cli.js"
       },
       "engines": {
-        "node": "^16.10.0 || ^18.0.0 || >=20.0.0"
+        "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0"
       },
       "peerDependencies": {
         "@babel/core": ">=7.0.0-beta.0 <8",
+        "@jest/transform": "^29.0.0",
         "@jest/types": "^29.0.0",
         "babel-jest": "^29.0.0",
         "jest": "^29.0.0",
@@ -16867,6 +17083,9 @@
         "@babel/core": {
           "optional": true
         },
+        "@jest/transform": {
+          "optional": true
+        },
         "@jest/types": {
           "optional": true
         },
@@ -16891,13 +17110,11 @@
       }
     },
     "node_modules/ts-jest/node_modules/semver": {
-      "version": "7.5.4",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
-      "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
+      "version": "7.6.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
+      "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
       "dev": true,
-      "dependencies": {
-        "lru-cache": "^6.0.0"
-      },
+      "license": "ISC",
       "bin": {
         "semver": "bin/semver.js"
       },
@@ -16906,15 +17123,17 @@
       }
     },
     "node_modules/ts-loader": {
-      "version": "9.4.2",
-      "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.4.2.tgz",
-      "integrity": "sha512-OmlC4WVmFv5I0PpaxYb+qGeGOdm5giHU7HwDDUjw59emP2UYMHy9fFSDcYgSNoH8sXcj4hGCSEhlDZ9ULeDraA==",
+      "version": "9.5.1",
+      "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.5.1.tgz",
+      "integrity": "sha512-rNH3sK9kGZcH9dYzC7CewQm4NtxJTjSEVRJ2DyBZR7f8/wcta+iV44UPCXc5+nzDzivKtlzV6c9P4e+oFhDLYg==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "chalk": "^4.1.0",
         "enhanced-resolve": "^5.0.0",
         "micromatch": "^4.0.0",
-        "semver": "^7.3.4"
+        "semver": "^7.3.4",
+        "source-map": "^0.7.4"
       },
       "engines": {
         "node": ">=12.0.0"
@@ -16997,6 +17216,16 @@
         "node": ">=10"
       }
     },
+    "node_modules/ts-loader/node_modules/source-map": {
+      "version": "0.7.4",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz",
+      "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==",
+      "dev": true,
+      "license": "BSD-3-Clause",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/ts-loader/node_modules/supports-color": {
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -17010,10 +17239,11 @@
       }
     },
     "node_modules/ts-node": {
-      "version": "10.9.1",
-      "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz",
-      "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==",
+      "version": "10.9.2",
+      "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
+      "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "@cspotcode/source-map-support": "^0.8.0",
         "@tsconfig/node10": "^1.0.7",
@@ -17066,9 +17296,10 @@
       }
     },
     "node_modules/tslib": {
-      "version": "2.6.3",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
-      "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ=="
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz",
+      "integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA==",
+      "license": "0BSD"
     },
     "node_modules/type-check": {
       "version": "0.4.0",
@@ -17183,16 +17414,17 @@
       }
     },
     "node_modules/typescript": {
-      "version": "4.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
-      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "version": "5.5.4",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz",
+      "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==",
       "dev": true,
+      "license": "Apache-2.0",
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
       },
       "engines": {
-        "node": ">=4.2.0"
+        "node": ">=14.17"
       }
     },
     "node_modules/typescript-eslint": {
@@ -17235,6 +17467,13 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/universalify": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
@@ -17293,6 +17532,7 @@
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/upper-case/-/upper-case-2.0.2.tgz",
       "integrity": "sha512-KgdgDGJt2TpuwBUIjgG6lzw2GWFRCW9Qkfkiv0DxqHHLYJHmtmdUIKcZd8rHgFSjopVTlw6ggzCm1b8MFQwikg==",
+      "license": "MIT",
       "dependencies": {
         "tslib": "^2.0.3"
       }
@@ -17301,6 +17541,7 @@
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/upper-case-first/-/upper-case-first-2.0.2.tgz",
       "integrity": "sha512-514ppYHBaKwfJRK/pNC6c/OxfGa0obSnAl106u97Ed0I625Nin96KAjttZF6ZL3e1XLtphxnqrOi9iWgm+u+bg==",
+      "license": "MIT",
       "dependencies": {
         "tslib": "^2.0.3"
       }
@@ -17331,9 +17572,10 @@
       "dev": true
     },
     "node_modules/use-sync-external-store": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz",
-      "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==",
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.2.tgz",
+      "integrity": "sha512-PElTlVMwpblvbNqQ82d2n6RjStvdSoNe9FG28kNfz3WiXilJm4DdNkEzRhCZuIDwY8U08WVihhGR5iRqAwfDiw==",
+      "license": "MIT",
       "peerDependencies": {
         "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       }
@@ -17360,9 +17602,10 @@
       }
     },
     "node_modules/uuid": {
-      "version": "7.0.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.2.tgz",
-      "integrity": "sha512-vy9V/+pKG+5ZTYKf+VcphF5Oc6EFiu3W8Nv3P3zIh0EqVI80ZxOzuPfe9EHjkFNvf8+xuTHVeei4Drydlx4zjw==",
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz",
+      "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==",
+      "license": "MIT",
       "bin": {
         "uuid": "dist/bin/uuid"
       }
@@ -18279,9 +18522,10 @@
       }
     },
     "node_modules/zrender": {
-      "version": "5.4.4",
-      "resolved": "https://registry.npmjs.org/zrender/-/zrender-5.4.4.tgz",
-      "integrity": "sha512-0VxCNJ7AGOMCWeHVyTrGzUgrK4asT4ml9PEkeGirAkKNYXYzoPJCLvmyfdoOXcjTHPs10OZVMfD1Rwg16AZyYw==",
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/zrender/-/zrender-5.6.0.tgz",
+      "integrity": "sha512-uzgraf4njmmHAbEUxMJ8Oxg+P3fT04O+9p7gY+wJRVxo8Ge+KmYv0WJev945EH4wFuc4OY2NLXz46FZrWS9xJg==",
+      "license": "BSD-3-Clause",
       "dependencies": {
         "tslib": "2.3.0"
       }
@@ -18289,23 +18533,29 @@
     "node_modules/zrender/node_modules/tslib": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz",
-      "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg=="
+      "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==",
+      "license": "0BSD"
     },
     "node_modules/zustand": {
-      "version": "4.3.2",
-      "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.3.2.tgz",
-      "integrity": "sha512-rd4haDmlwMTVWVqwvgy00ny8rtti/klRoZjFbL/MAcDnmD5qSw/RZc+Vddstdv90M5Lv6RPgWvm1Hivyn0QgJw==",
+      "version": "4.5.5",
+      "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.5.tgz",
+      "integrity": "sha512-+0PALYNJNgK6hldkgDq2vLrw5f6g/jCInz52n9RTpropGgeAf/ioFUCdtsjCqu4gNhW9D01rUQBROoRjdzyn2Q==",
+      "license": "MIT",
       "dependencies": {
-        "use-sync-external-store": "1.2.0"
+        "use-sync-external-store": "1.2.2"
       },
       "engines": {
         "node": ">=12.7.0"
       },
       "peerDependencies": {
-        "immer": ">=9.0",
+        "@types/react": ">=16.8",
+        "immer": ">=9.0.6",
         "react": ">=16.8"
       },
       "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
         "immer": {
           "optional": true
         },
@@ -18807,20 +19057,27 @@
       "dev": true
     },
     "@blueprintjs/colors": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/colors/-/colors-5.1.1.tgz",
-      "integrity": "sha512-Mni/GgCYLaMf5U5zsCN42skOG49w3U0QmUFuJgFZ/1pv+3cHF/9xR4L4dXcj5DtgJoHkmUbK36PR5mdFB65WEA==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/colors/-/colors-5.1.2.tgz",
+      "integrity": "sha512-7CWwVsXK4YTN9Z/wkjnS3p7VE8YfIXXv2UaySAbtcw6rBkmoSHjLRtfohSA5yNy8xYTQ4KY2odKZSUW0W/Nltw==",
       "requires": {
         "tslib": "~2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.6.3",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+          "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ=="
+        }
       }
     },
     "@blueprintjs/core": {
-      "version": "5.10.5",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-5.10.5.tgz",
-      "integrity": "sha512-mvTB+Ox31nt9830hEgVOi/A53QI71hpOzV/7i3ZgzzrYAaEGgFMcwa7X0kn9PSOGfxw87nWL9VpYSirnzpC6lw==",
+      "version": "5.13.1",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-5.13.1.tgz",
+      "integrity": "sha512-PDZ9X/xGBetwU2AqQuCDGVWZZMmt6+/BCnmoKXxsBBZIuWQuHDiKcwvb9rzup0htsA6P7KGl5aw7ocmDvZPpBw==",
       "requires": {
-        "@blueprintjs/colors": "^5.1.1",
-        "@blueprintjs/icons": "^5.10.0",
+        "@blueprintjs/colors": "^5.1.2",
+        "@blueprintjs/icons": "^5.13.0",
         "@popperjs/core": "^2.11.8",
         "classnames": "^2.3.1",
         "normalize.css": "^8.0.1",
@@ -18829,31 +19086,45 @@
         "react-uid": "^2.3.3",
         "tslib": "~2.6.2",
         "use-sync-external-store": "^1.2.0"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.6.3",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+          "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ=="
+        }
       }
     },
     "@blueprintjs/datetime": {
-      "version": "5.3.7",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/datetime/-/datetime-5.3.7.tgz",
-      "integrity": "sha512-6rDMqSU8jeiAlMxS30MYAQc4X1sfMGWwFjfs3IoKxJfcfieN+ytus5G+if6zjsSKUjdhL9+qBTgKRo3VJJEQ3g==",
+      "version": "5.3.11",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/datetime/-/datetime-5.3.11.tgz",
+      "integrity": "sha512-6MMictO0OPc6gRoPt/gZ6V5tFTKj9d2M3Mpn4alNa74uZF/Y1FjBfOvSPOsV7JajBOVjqiaSEBRZIFHrhRr91Q==",
       "requires": {
-        "@blueprintjs/core": "^5.10.5",
-        "@blueprintjs/icons": "^5.10.0",
-        "@blueprintjs/select": "^5.2.1",
+        "@blueprintjs/core": "^5.13.1",
+        "@blueprintjs/icons": "^5.13.0",
+        "@blueprintjs/select": "^5.2.5",
         "classnames": "^2.3.1",
         "date-fns": "^2.28.0",
         "date-fns-tz": "^2.0.0",
         "react-day-picker": "7.4.9",
         "tslib": "~2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.6.3",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+          "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ=="
+        }
       }
     },
     "@blueprintjs/datetime2": {
-      "version": "2.3.7",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/datetime2/-/datetime2-2.3.7.tgz",
-      "integrity": "sha512-Q6IBo5hcvoDtH9FsFaysZhadXNS6iyhuyaRY+rJD3y3qLaUt6wiYDhw5LN6L4wuasbPO1g0lVDwY6bkGuIG4jg==",
+      "version": "2.3.11",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/datetime2/-/datetime2-2.3.11.tgz",
+      "integrity": "sha512-o9NYfUAb8xYiGZQ56tn6CXm8jDNxwSxJvZgrzs6EVYCuCFrsBr9WAjy0p768ZbZlzuWwClbEjslr8s+E6etU/w==",
       "requires": {
-        "@blueprintjs/core": "^5.10.5",
-        "@blueprintjs/datetime": "^5.3.7",
-        "@blueprintjs/icons": "^5.10.0",
+        "@blueprintjs/core": "^5.13.1",
+        "@blueprintjs/datetime": "^5.3.11",
+        "@blueprintjs/icons": "^5.13.0",
         "classnames": "^2.3.1",
         "date-fns": "^2.28.0",
         "react-day-picker": "^8.10.0",
@@ -18865,28 +19136,47 @@
           "version": "8.10.1",
           "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-8.10.1.tgz",
           "integrity": "sha512-TMx7fNbhLk15eqcMt+7Z7S2KF7mfTId/XJDjKE8f+IUcFn0l08/kI4FiYTL/0yuOLmEcbR4Fwe3GJf/NiiMnPA=="
+        },
+        "tslib": {
+          "version": "2.6.3",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+          "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ=="
         }
       }
     },
     "@blueprintjs/icons": {
-      "version": "5.10.0",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-5.10.0.tgz",
-      "integrity": "sha512-Qp9H2U2/m6+p0Lh9ACuF2O6o0uVINMFimo+EgasgGaLx8SkJsl7ApB3hYQ+iRSYlQWotnoqd4Vtzj4c7sKukHQ==",
+      "version": "5.13.0",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-5.13.0.tgz",
+      "integrity": "sha512-L096dBjzfnWW7fWXM311S2C/5Zn0EuEK9q6G84QvWP0BZJOTowU1EIWLj90IgGtNajld/3ZUAj6eJf+ryt/kjQ==",
       "requires": {
         "change-case": "^4.1.2",
         "classnames": "^2.3.1",
         "tslib": "~2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.6.3",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+          "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ=="
+        }
       }
     },
     "@blueprintjs/select": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/select/-/select-5.2.1.tgz",
-      "integrity": "sha512-rXM9ik2mLd394g+sxwbHv2bGfrLbcdY9gFftqmJmm3vu7XJ2kRrsC1oOlNSC4dNBpUdHnyAzBTHEu81B1LkRbQ==",
+      "version": "5.2.5",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/select/-/select-5.2.5.tgz",
+      "integrity": "sha512-mO9r5iQ4uxdgScTyNSNSLq3DUZgFtI3dAO1VRUNONBGPcSROVi+3ixYYynwcad18ftAZz/WuUD9ZHqVyDwXmbQ==",
       "requires": {
-        "@blueprintjs/core": "^5.10.5",
-        "@blueprintjs/icons": "^5.10.0",
+        "@blueprintjs/core": "^5.13.1",
+        "@blueprintjs/icons": "^5.13.0",
         "classnames": "^2.3.1",
         "tslib": "~2.6.2"
+      },
+      "dependencies": {
+        "tslib": {
+          "version": "2.6.3",
+          "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
+          "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ=="
+        }
       }
     },
     "@cspotcode/source-map-support": {
@@ -19291,9 +19581,9 @@
       "dev": true
     },
     "@druid-toolkit/query": {
-      "version": "0.22.22",
-      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.22.tgz",
-      "integrity": "sha512-QCzIFr0H6uk5KsGEKO/Vk1fWWurf54ENagFCygsKVk3p2sCRmpZ7WGKeQ5i3AFh2847tOeopZI0OniEv2yTLGQ==",
+      "version": "0.22.23",
+      "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.23.tgz",
+      "integrity": "sha512-yQOUAQJP63rzsTCdLcqNB8aRtsYPw8rYBfPSXc4zfAA4y/GJc9OJeHcLFRMdUtpwBtm0ueARMUlTSQcTsyV8gQ==",
       "requires": {
         "tslib": "^2.5.2"
       }
@@ -19513,9 +19803,9 @@
       }
     },
     "@fontsource/open-sans": {
-      "version": "5.0.28",
-      "resolved": "https://registry.npmjs.org/@fontsource/open-sans/-/open-sans-5.0.28.tgz",
-      "integrity": "sha512-hBvJHY76pJT/JynGUB5EXWhnzjYfLdcMn655J5p1v9lTT9HdQSy+keq2KPVXO2Htlg998BBa3p6u/jlrZ6w0kg=="
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/@fontsource/open-sans/-/open-sans-5.1.0.tgz",
+      "integrity": "sha512-g+mjF8gWUDwck9DrRCkhmFeEj7fskjtKZJKAQguVzSg93lc6ThakTHMRgs0dZfe5qBbktrV839tDrb4bIDyZSA=="
     },
     "@humanfs/core": {
       "version": "0.19.0",
@@ -20345,15 +20635,15 @@
       }
     },
     "@testing-library/dom": {
-      "version": "9.3.0",
-      "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-9.3.0.tgz",
-      "integrity": "sha512-Dffe68pGwI6WlLRYR2I0piIkyole9cSBH5jGQKCGMRpHW5RHCqAUaqc2Kv0tUyd4dU4DLPKhJIjyKOnjv4tuUw==",
+      "version": "10.4.0",
+      "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.0.tgz",
+      "integrity": "sha512-pemlzrSESWbdAloYml3bAJMEfNh1Z7EduzqPKprCH5S341frlpYnUEW0H72dLxa6IsYr+mPno20GiSm+h9dEdQ==",
       "dev": true,
       "requires": {
         "@babel/code-frame": "^7.10.4",
         "@babel/runtime": "^7.12.5",
         "@types/aria-query": "^5.0.1",
-        "aria-query": "^5.0.0",
+        "aria-query": "5.3.0",
         "chalk": "^4.1.0",
         "dom-accessibility-api": "^0.5.9",
         "lz-string": "^1.5.0",
@@ -20412,14 +20702,12 @@
       }
     },
     "@testing-library/react": {
-      "version": "14.0.0",
-      "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-14.0.0.tgz",
-      "integrity": "sha512-S04gSNJbYE30TlIMLTzv6QCTzt9AqIF5y6s6SzVFILNcNvbV/jU96GeiTPillGQo+Ny64M/5PV7klNYYgv5Dfg==",
+      "version": "16.0.1",
+      "resolved": "https://registry.npmjs.org/@testing-library/react/-/react-16.0.1.tgz",
+      "integrity": "sha512-dSmwJVtJXmku+iocRhWOUFbrERC76TX2Mnf0ATODz8brzAZrMBbzLwQixlBSanZxR6LddK3eiwpSFZgDET1URg==",
       "dev": true,
       "requires": {
-        "@babel/runtime": "^7.12.5",
-        "@testing-library/dom": "^9.0.0",
-        "@types/react-dom": "^18.0.0"
+        "@babel/runtime": "^7.12.5"
       }
     },
     "@tootallnate/once": {
@@ -20453,9 +20741,9 @@
       "dev": true
     },
     "@types/aria-query": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.1.tgz",
-      "integrity": "sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q==",
+      "version": "5.0.4",
+      "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz",
+      "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==",
       "dev": true
     },
     "@types/babel__core": {
@@ -20553,33 +20841,33 @@
       }
     },
     "@types/d3-array": {
-      "version": "2.12.3",
-      "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-2.12.3.tgz",
-      "integrity": "sha512-hN879HLPTVqZV3FQEXy7ptt083UXwguNbnxdTGzVW4y4KjX5uyNKljrQixZcSJfLyFirbpUokxpXtvR+N5+KIg==",
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz",
+      "integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==",
       "dev": true
     },
     "@types/d3-axis": {
-      "version": "2.1.3",
-      "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-2.1.3.tgz",
-      "integrity": "sha512-QjXjwZ0xzyrW2ndkmkb09ErgWDEYtbLBKGui73QLMFm3woqWpxptfD5Y7vqQdybMcu7WEbjZ5q+w2w5+uh2IjA==",
+      "version": "3.0.6",
+      "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz",
+      "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==",
       "dev": true,
       "requires": {
-        "@types/d3-selection": "^2"
+        "@types/d3-selection": "*"
       }
     },
     "@types/d3-dsv": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-2.0.3.tgz",
-      "integrity": "sha512-15sp4Z+ZVWuZuV0QEDu4cu/0C5vlD+JYXaUMDs8JTWpTJjcrAtjyR1vVwEfbgmU5kLNOOMRTlDCYyWWFx7eh/w==",
+      "version": "3.0.7",
+      "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz",
+      "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==",
       "dev": true
     },
     "@types/d3-scale": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-3.3.2.tgz",
-      "integrity": "sha512-gGqr7x1ost9px3FvIfUMi5XA/F/yAf4UkUDtdQhpH92XCT0Oa7zkkRzY61gPVJq+DxpHn/btouw5ohWkbBsCzQ==",
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.8.tgz",
+      "integrity": "sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==",
       "dev": true,
       "requires": {
-        "@types/d3-time": "^2"
+        "@types/d3-time": "*"
       }
     },
     "@types/d3-scale-chromatic": {
@@ -20589,9 +20877,9 @@
       "dev": true
     },
     "@types/d3-selection": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-2.0.1.tgz",
-      "integrity": "sha512-3mhtPnGE+c71rl/T5HMy+ykg7migAZ4T6gzU0HxpgBFKcasBrSnwRbYV1/UZR6o5fkpySxhWxAhd7yhjj8jL7g==",
+      "version": "3.0.11",
+      "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz",
+      "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==",
       "dev": true
     },
     "@types/d3-time": {
@@ -20601,13 +20889,13 @@
       "dev": true
     },
     "@types/enzyme": {
-      "version": "3.10.17",
-      "resolved": "https://registry.npmjs.org/@types/enzyme/-/enzyme-3.10.17.tgz",
-      "integrity": "sha512-4+CMYPx8cToXnF1WxUdjKCZFSrslJqkIW5N5D27stzJsnJmw4dWbOcmhPERJpe+CufV0LanV7BgTWuj63Z2/Vg==",
+      "version": "3.10.18",
+      "resolved": "https://registry.npmjs.org/@types/enzyme/-/enzyme-3.10.18.tgz",
+      "integrity": "sha512-RaO/TyyHZvXkpzinbMTZmd/S5biU4zxkvDsn22ujC29t9FMSzq8tnn8f2MxQ2P8GVhFRG5jTAL05DXKyTtpEQQ==",
       "dev": true,
       "requires": {
         "@types/cheerio": "*",
-        "@types/react": "^18.2.42"
+        "@types/react": "^18.3.11"
       }
     },
     "@types/enzyme-adapter-react-16": {
@@ -20755,18 +21043,18 @@
       "dev": true
     },
     "@types/lodash.debounce": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/@types/lodash.debounce/-/lodash.debounce-4.0.6.tgz",
-      "integrity": "sha512-4WTmnnhCfDvvuLMaF3KV4Qfki93KebocUF45msxhYyjMttZDQYzHkO639ohhk8+oco2cluAFL3t5+Jn4mleylQ==",
+      "version": "4.0.9",
+      "resolved": "https://registry.npmjs.org/@types/lodash.debounce/-/lodash.debounce-4.0.9.tgz",
+      "integrity": "sha512-Ma5JcgTREwpLRwMM+XwBR7DaWe96nC38uCBDFKZWbNKD+osjVzdpnUSwBcqCptrp16sSOLBAUb50Car5I0TCsQ==",
       "dev": true,
       "requires": {
         "@types/lodash": "*"
       }
     },
     "@types/lodash.escape": {
-      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/@types/lodash.escape/-/lodash.escape-4.0.6.tgz",
-      "integrity": "sha512-/CI97B3wf1R4oD4yotsGoX/5bobL/RC8olTQ16iunzdjufcdD8GyIvNDatg1IfzVKaFBZmxuY0+WQDpdaQHLzg==",
+      "version": "4.0.9",
+      "resolved": "https://registry.npmjs.org/@types/lodash.escape/-/lodash.escape-4.0.9.tgz",
+      "integrity": "sha512-YR9R6RU9D36OWrhVZ+r4fY8mMXQtofnLXAcE6CkKIxdH66uiCdEjkhxx/eiJnYpxjOFmPbWBUUvdv8lckO823Q==",
       "dev": true,
       "requires": {
         "@types/lodash": "*"
@@ -20785,10 +21073,13 @@
       "dev": true
     },
     "@types/node": {
-      "version": "12.11.7",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-12.11.7.tgz",
-      "integrity": "sha512-JNbGaHFCLwgHn/iCckiGSOZ1XYHsKFwREtzPwSGCVld1SGhOlmZw2D4ZI94HQCrBHbADzW9m4LER/8olJTRGHA==",
-      "dev": true
+      "version": "18.19.56",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.56.tgz",
+      "integrity": "sha512-4EMJlWwwGnVPflJAtM14p9eVSa6BOv5b92mCsh5zcM1UagNtEtrbbtaE6WE1tw2TabavatnwqXjlIpcAEuJJNg==",
+      "dev": true,
+      "requires": {
+        "undici-types": "~5.26.4"
+      }
     },
     "@types/node-forge": {
       "version": "1.3.10",
@@ -20800,9 +21091,9 @@
       }
     },
     "@types/numeral": {
-      "version": "0.0.26",
-      "resolved": "https://registry.npmjs.org/@types/numeral/-/numeral-0.0.26.tgz",
-      "integrity": "sha512-DwCsRqeOWopdEsm5KLTxKVKDSDoj+pzZD1vlwu1GQJ6IF3RhjuleYlRwyRH6MJLGaf3v8wFTnC6wo3yYfz0bnA==",
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@types/numeral/-/numeral-2.0.5.tgz",
+      "integrity": "sha512-kH8I7OSSwQu9DS9JYdFWbuvhVzvFRoCPCkGxNwoGgaPeDfEPJlcxNvEOypZhQ3XXHsGbfIuYcxcJxKUfJHnRfw==",
       "dev": true
     },
     "@types/parse-json": {
@@ -20828,22 +21119,21 @@
       "dev": true
     },
     "@types/react": {
-      "version": "18.2.42",
-      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.42.tgz",
-      "integrity": "sha512-c1zEr96MjakLYus/wPnuWDo1/zErfdU9rNsIGmE+NV71nx88FG9Ttgo5dqorXTu/LImX2f63WBP986gJkMPNbA==",
+      "version": "18.3.11",
+      "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.11.tgz",
+      "integrity": "sha512-r6QZ069rFTjrEYgFdOck1gK7FLVsgJE7tTz0pQBczlBNUhBNk0MQH4UbnFSwjpQLMkLzgqvBBa+qGpLje16eTQ==",
       "requires": {
         "@types/prop-types": "*",
-        "@types/scheduler": "*",
         "csstype": "^3.0.2"
       }
     },
     "@types/react-dom": {
-      "version": "18.2.17",
-      "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.2.17.tgz",
-      "integrity": "sha512-rvrT/M7Df5eykWFxn6MYt5Pem/Dbyc1N8Y0S9Mrkw2WFCRiqUgw9P7ul2NpwsXCSM1DVdENzdG9J5SreqfAIWg==",
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.1.tgz",
+      "integrity": "sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ==",
       "dev": true,
       "requires": {
-        "@types/react": "^18.2.42"
+        "@types/react": "^18.3.11"
       }
     },
     "@types/react-router": {
@@ -20853,7 +21143,7 @@
       "dev": true,
       "requires": {
         "@types/history": "*",
-        "@types/react": "^18.2.42"
+        "@types/react": "^18.3.11"
       }
     },
     "@types/react-router-dom": {
@@ -20863,7 +21153,7 @@
       "dev": true,
       "requires": {
         "@types/history": "^4.7.11",
-        "@types/react": "^18.2.42",
+        "@types/react": "^18.3.11",
         "@types/react-router": "*"
       }
     },
@@ -20873,7 +21163,7 @@
       "integrity": "sha512-J3NKVdPguGcSSN41/EDV3BYdYfndxUil2SX4wDBJiVfcnyopCegdhIrC2kcaa4CxjpaPChe1bja2k8LLDxP0ZQ==",
       "dev": true,
       "requires": {
-        "@types/react": "^18.2.42"
+        "@types/react": "^18.3.11"
       }
     },
     "@types/react-table": {
@@ -20881,7 +21171,7 @@
       "resolved": "https://registry.npmjs.org/@types/react-table/-/react-table-6.8.5.tgz",
       "integrity": "sha512-ueCsAadG1IwuuAZM+MWf2SoxbccSWweyQa9YG6xGN5cOVK3SayPOJW4MsUHGpY0V/Q+iZWgohpasliiao29O6g==",
       "requires": {
-        "@types/react": "^18.2.42"
+        "@types/react": "^18.3.11"
       }
     },
     "@types/retry": {
@@ -20890,11 +21180,6 @@
       "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",
       "dev": true
     },
-    "@types/scheduler": {
-      "version": "0.16.8",
-      "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.8.tgz",
-      "integrity": "sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A=="
-    },
     "@types/send": {
       "version": "0.17.4",
       "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
@@ -21346,9 +21631,9 @@
       }
     },
     "ace-builds": {
-      "version": "1.4.14",
-      "resolved": "https://registry.npmjs.org/ace-builds/-/ace-builds-1.4.14.tgz",
-      "integrity": "sha512-NBOQlm9+7RBqRqZwimpgquaLeTJFayqb9UEPtTkpC3TkkwDnlsT/TwsCC0svjt9kEZ6G9mH5AEOHSz6Q/HrzQQ=="
+      "version": "1.5.3",
+      "resolved": "https://registry.npmjs.org/ace-builds/-/ace-builds-1.5.3.tgz",
+      "integrity": "sha512-WN5BKR2aTSuBmisO8jo3Fytk6sOmJGki82v/Boeic81IgYN8pFHNkXq2anDF0XkmfDWMqLbRoW9sjc/GtKzQbQ=="
     },
     "acorn": {
       "version": "8.13.0",
@@ -21526,46 +21811,12 @@
       "dev": true
     },
     "aria-query": {
-      "version": "5.1.3",
-      "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.1.3.tgz",
-      "integrity": "sha512-R5iJ5lkuHybztUfuOAznmboyjWq8O6sqNqtK7CLOqdydi54VNbORp49mb14KbWgG1QD3JFO9hJdZ+y4KutfdOQ==",
+      "version": "5.3.0",
+      "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz",
+      "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==",
       "dev": true,
       "requires": {
-        "deep-equal": "^2.0.5"
-      },
-      "dependencies": {
-        "deep-equal": {
-          "version": "2.2.1",
-          "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-2.2.1.tgz",
-          "integrity": "sha512-lKdkdV6EOGoVn65XaOsPdH4rMxTZOnmFyuIkMjM1i5HHCbfjC97dawgTAy0deYNfuqUqW+Q5VrVaQYtUpSd6yQ==",
-          "dev": true,
-          "requires": {
-            "array-buffer-byte-length": "^1.0.0",
-            "call-bind": "^1.0.2",
-            "es-get-iterator": "^1.1.3",
-            "get-intrinsic": "^1.2.0",
-            "is-arguments": "^1.1.1",
-            "is-array-buffer": "^3.0.2",
-            "is-date-object": "^1.0.5",
-            "is-regex": "^1.1.4",
-            "is-shared-array-buffer": "^1.0.2",
-            "isarray": "^2.0.5",
-            "object-is": "^1.1.5",
-            "object-keys": "^1.1.1",
-            "object.assign": "^4.1.4",
-            "regexp.prototype.flags": "^1.5.0",
-            "side-channel": "^1.0.4",
-            "which-boxed-primitive": "^1.0.2",
-            "which-collection": "^1.0.1",
-            "which-typed-array": "^1.1.9"
-          }
-        },
-        "isarray": {
-          "version": "2.0.5",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz",
-          "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==",
-          "dev": true
-        }
+        "dequal": "^2.0.3"
       }
     },
     "array-buffer-byte-length": {
@@ -21703,6 +21954,12 @@
       "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
       "dev": true
     },
+    "async": {
+      "version": "3.2.6",
+      "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz",
+      "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==",
+      "dev": true
+    },
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
@@ -21740,9 +21997,9 @@
       }
     },
     "axios": {
-      "version": "1.7.4",
-      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz",
-      "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==",
+      "version": "1.7.7",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz",
+      "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==",
       "requires": {
         "follow-redirects": "^1.15.6",
         "form-data": "^4.0.0",
@@ -22293,7 +22550,8 @@
     "commander": {
       "version": "2.20.0",
       "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.0.tgz",
-      "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ=="
+      "integrity": "sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==",
+      "dev": true
     },
     "compressible": {
       "version": "2.0.18",
@@ -22396,18 +22654,13 @@
       "dev": true
     },
     "copy-to-clipboard": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.2.0.tgz",
-      "integrity": "sha512-eOZERzvCmxS8HWzugj4Uxl8OJxa7T2k1Gi0X5qavwydHIfuSHq2dTD09LOg/XyGq4Zpb5IsR/2OJ5lbOegz78w==",
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/copy-to-clipboard/-/copy-to-clipboard-3.3.3.tgz",
+      "integrity": "sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==",
       "requires": {
         "toggle-selection": "^1.0.6"
       }
     },
-    "core-js": {
-      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.10.1.tgz",
-      "integrity": "sha512-pwCxEXnj27XG47mu7SXAwhLP3L5CrlvCB91ANUkIz40P27kUcvNfSdvyZJ9CLHiVoKSp+TTChMQMSKQEH/IQxA=="
-    },
     "core-util-is": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
@@ -22669,31 +22922,46 @@
       "integrity": "sha512-KxnUB0ZMlnUWCsx2Z8MUsr6qV6ja1w9ArPErJaJaF8a5SOWoHLIszeCTKGRGRgtLgYrs1E8CHkNSP1VZTTPc9g=="
     },
     "d3-array": {
-      "version": "2.12.1",
-      "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz",
-      "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==",
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz",
+      "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==",
       "requires": {
-        "internmap": "^1.0.0"
+        "internmap": "1 - 2"
       }
     },
     "d3-axis": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-2.1.0.tgz",
-      "integrity": "sha512-z/G2TQMyuf0X3qP+Mh+2PimoJD41VOCjViJzT0BHeL/+JQAofkiWZbWxlwFGb1N8EN+Cl/CW+MUKbVzr1689Cw=="
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz",
+      "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw=="
     },
     "d3-color": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-2.0.0.tgz",
-      "integrity": "sha512-SPXi0TSKPD4g9tw0NMZFnR95XVgUZiBH+uUTqQuDu1OsE2zomHU7ho0FISciaPvosimixwHFl3WHLGabv6dDgQ=="
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz",
+      "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA=="
     },
     "d3-dsv": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-2.0.0.tgz",
-      "integrity": "sha512-E+Pn8UJYx9mViuIUkoc93gJGGYut6mSDKy2+XaPwccwkRGlR+LO97L2VCCRjQivTwLHkSnAJG7yo00BWY6QM+w==",
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz",
+      "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==",
       "requires": {
-        "commander": "2",
-        "iconv-lite": "0.4",
+        "commander": "7",
+        "iconv-lite": "0.6",
         "rw": "1"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz",
+          "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw=="
+        },
+        "iconv-lite": {
+          "version": "0.6.3",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+          "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3.0.0"
+          }
+        }
       }
     },
     "d3-format": {
@@ -22702,25 +22970,33 @@
       "integrity": "sha512-TUswGe6hfguUX1CtKxyG2nymO+1lyThbkS1ifLX0Sr+dOQtAD5gkrffpHnx+yHNKUZ0Bmg5T4AjUQwugPDrm0g=="
     },
     "d3-interpolate": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-2.0.1.tgz",
-      "integrity": "sha512-c5UhwwTs/yybcmTpAVqwSFl6vrQ8JZJoT5F7xNFK9pymv5C0Ymcc9/LIJHtYIggg/yS9YHw8i8O8tgb9pupjeQ==",
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz",
+      "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==",
       "requires": {
-        "d3-color": "1 - 2"
+        "d3-color": "1 - 3"
       }
     },
     "d3-scale": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-3.3.0.tgz",
-      "integrity": "sha512-1JGp44NQCt5d1g+Yy+GeOnZP7xHo0ii8zsQp6PGzd+C1/dl0KGsp9A7Mxwp+1D1o4unbTTxVdU/ZOIEBoeZPbQ==",
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz",
+      "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==",
       "requires": {
-        "d3-array": "^2.3.0",
-        "d3-format": "1 - 2",
-        "d3-interpolate": "1.2.0 - 2",
-        "d3-time": "^2.1.1",
-        "d3-time-format": "2 - 3"
+        "d3-array": "2.10.0 - 3",
+        "d3-format": "1 - 3",
+        "d3-interpolate": "1.2.0 - 3",
+        "d3-time": "2.1.1 - 3",
+        "d3-time-format": "2 - 4"
       },
       "dependencies": {
+        "d3-array": {
+          "version": "2.12.1",
+          "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz",
+          "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==",
+          "requires": {
+            "internmap": "^1.0.0"
+          }
+        },
         "d3-time": {
           "version": "2.1.1",
           "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-2.1.1.tgz",
@@ -22741,9 +23017,9 @@
       }
     },
     "d3-selection": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-2.0.0.tgz",
-      "integrity": "sha512-XoGGqhLUN/W14NmaqcO/bb1nqjDAw5WtSYb2X8wiuQWvSZUsUVYsOSkOybUrNvcBjaywBdYPy03eXHMXjk9nZA=="
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz",
+      "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ=="
     },
     "d3-time": {
       "version": "1.1.0",
@@ -22922,6 +23198,12 @@
       "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak=",
       "dev": true
     },
+    "dequal": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz",
+      "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==",
+      "dev": true
+    },
     "destroy": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
@@ -23030,12 +23312,12 @@
       "dev": true
     },
     "echarts": {
-      "version": "5.4.3",
-      "resolved": "https://registry.npmjs.org/echarts/-/echarts-5.4.3.tgz",
-      "integrity": "sha512-mYKxLxhzy6zyTi/FaEbJMOZU1ULGEQHaeIeuMR5L+JnJTpz+YR03mnnpBhbR4+UYJAgiXgpyTVLffPAjOTLkZA==",
+      "version": "5.5.1",
+      "resolved": "https://registry.npmjs.org/echarts/-/echarts-5.5.1.tgz",
+      "integrity": "sha512-Fce8upazaAXUVUVsjgV6mBnGuqgO+JNDlcgF79Dksy4+wgGpQB2lmYoO4TSweFg/mZITdpGHomw/cNBJZj1icA==",
       "requires": {
         "tslib": "2.3.0",
-        "zrender": "5.4.4"
+        "zrender": "5.6.0"
       },
       "dependencies": {
         "tslib": {
@@ -23051,6 +23333,15 @@
       "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
       "dev": true
     },
+    "ejs": {
+      "version": "3.1.10",
+      "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz",
+      "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==",
+      "dev": true,
+      "requires": {
+        "jake": "^10.8.5"
+      }
+    },
     "electron-to-chromium": {
       "version": "1.5.33",
       "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.33.tgz",
@@ -23189,31 +23480,6 @@
       "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
       "dev": true
     },
-    "es-get-iterator": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz",
-      "integrity": "sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw==",
-      "dev": true,
-      "requires": {
-        "call-bind": "^1.0.2",
-        "get-intrinsic": "^1.1.3",
-        "has-symbols": "^1.0.3",
-        "is-arguments": "^1.1.1",
-        "is-map": "^2.0.2",
-        "is-set": "^2.0.2",
-        "is-string": "^1.0.7",
-        "isarray": "^2.0.5",
-        "stop-iteration-iterator": "^1.0.0"
-      },
-      "dependencies": {
-        "isarray": {
-          "version": "2.0.5",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz",
-          "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==",
-          "dev": true
-        }
-      }
-    },
     "es-iterator-helpers": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.1.0.tgz",
@@ -24031,6 +24297,35 @@
       "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-2.0.5.tgz",
       "integrity": "sha512-P9bmyZ3h/PRG+Nzga+rbdI4OEpNDzAVyy74uVO9ATgzLK6VtAsYybF/+TOCvrc0MO793d6+42lLyZTw7/ArVzA=="
     },
+    "filelist": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz",
+      "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==",
+      "dev": true,
+      "requires": {
+        "minimatch": "^5.0.1"
+      },
+      "dependencies": {
+        "brace-expansion": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+          "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+          "dev": true,
+          "requires": {
+            "balanced-match": "^1.0.0"
+          }
+        },
+        "minimatch": {
+          "version": "5.1.6",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
+          "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
+          "dev": true,
+          "requires": {
+            "brace-expansion": "^2.0.1"
+          }
+        }
+      }
+    },
     "fill-range": {
       "version": "7.1.1",
       "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
@@ -24675,6 +24970,7 @@
       "version": "0.4.24",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
       "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "dev": true,
       "requires": {
         "safer-buffer": ">= 2.1.2 < 3"
       }
@@ -24799,16 +25095,6 @@
       "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
       "dev": true
     },
-    "is-arguments": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz",
-      "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==",
-      "dev": true,
-      "requires": {
-        "call-bind": "^1.0.2",
-        "has-tostringtag": "^1.0.0"
-      }
-    },
     "is-array-buffer": {
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz",
@@ -25228,6 +25514,78 @@
         "set-function-name": "^2.0.1"
       }
     },
+    "jake": {
+      "version": "10.9.2",
+      "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz",
+      "integrity": "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==",
+      "dev": true,
+      "requires": {
+        "async": "^3.2.3",
+        "chalk": "^4.0.2",
+        "filelist": "^1.0.4",
+        "minimatch": "^3.1.2"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
+          "dev": true,
+          "requires": {
+            "color-convert": "^2.0.1"
+          }
+        },
+        "chalk": {
+          "version": "4.1.2",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+          "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.1.0",
+            "supports-color": "^7.1.0"
+          }
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "has-flag": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
+          "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
+          "dev": true
+        },
+        "minimatch": {
+          "version": "3.1.2",
+          "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+          "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+          "dev": true,
+          "requires": {
+            "brace-expansion": "^1.1.7"
+          }
+        },
+        "supports-color": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+          "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^4.0.0"
+          }
+        }
+      }
+    },
     "jest": {
       "version": "29.7.0",
       "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz",
@@ -27081,9 +27439,9 @@
       "dev": true
     },
     "make-error": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.5.tgz",
-      "integrity": "sha512-c3sIjNUow0+8swNwVpqoH4YCShKNFkMaw6oH1mNS2haDZQqkeZFlHS3dhoeEbKKmJB4vXpJucU6oH75aDYeE9g==",
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
+      "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
       "dev": true
     },
     "makeerror": {
@@ -27123,9 +27481,9 @@
       }
     },
     "memoize-one": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.1.1.tgz",
-      "integrity": "sha512-HKeeBpWvqiVJD57ZUAsJNm71eHTykffzcLZVYWiVfQeI1rJtuEaS7hQiEpWfVVk18donPwJEcFKIkCmPJNOhHA=="
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz",
+      "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q=="
     },
     "meow": {
       "version": "13.2.0",
@@ -27417,16 +27775,6 @@
       "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==",
       "dev": true
     },
-    "object-is": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz",
-      "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==",
-      "dev": true,
-      "requires": {
-        "call-bind": "^1.0.2",
-        "define-properties": "^1.1.3"
-      }
-    },
     "object-keys": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
@@ -28357,9 +28705,9 @@
       }
     },
     "react": {
-      "version": "18.2.0",
-      "resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
-      "integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
+      "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==",
       "requires": {
         "loose-envify": "^1.1.0"
       }
@@ -28398,12 +28746,12 @@
       }
     },
     "react-dom": {
-      "version": "18.2.0",
-      "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz",
-      "integrity": "sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==",
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz",
+      "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==",
       "requires": {
         "loose-envify": "^1.1.0",
-        "scheduler": "^0.23.0"
+        "scheduler": "^0.23.2"
       }
     },
     "react-fast-compare": {
@@ -28883,9 +29231,9 @@
       }
     },
     "scheduler": {
-      "version": "0.23.0",
-      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz",
-      "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==",
+      "version": "0.23.2",
+      "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz",
+      "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==",
       "requires": {
         "loose-envify": "^1.1.0"
       }
@@ -29433,15 +29781,6 @@
       "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=",
       "dev": true
     },
-    "stop-iteration-iterator": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz",
-      "integrity": "sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ==",
-      "dev": true,
-      "requires": {
-        "internal-slot": "^1.0.4"
-      }
-    },
     "stream-events": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz",
@@ -30029,19 +30368,20 @@
       "dev": true
     },
     "ts-jest": {
-      "version": "29.1.2",
-      "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.1.2.tgz",
-      "integrity": "sha512-br6GJoH/WUX4pu7FbZXuWGKGNDuU7b8Uj77g/Sp7puZV6EXzuByl6JrECvm0MzVzSTkSHWTihsXt+5XYER5b+g==",
+      "version": "29.2.5",
+      "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.5.tgz",
+      "integrity": "sha512-KD8zB2aAZrcKIdGk4OwpJggeLcH1FgrICqDSROWqlnJXGCXK4Mn6FcdK2B6670Xr73lHMG1kHw8R87A0ecZ+vA==",
       "dev": true,
       "requires": {
-        "bs-logger": "0.x",
-        "fast-json-stable-stringify": "2.x",
+        "bs-logger": "^0.2.6",
+        "ejs": "^3.1.10",
+        "fast-json-stable-stringify": "^2.1.0",
         "jest-util": "^29.0.0",
         "json5": "^2.2.3",
-        "lodash.memoize": "4.x",
-        "make-error": "1.x",
-        "semver": "^7.5.3",
-        "yargs-parser": "^21.0.1"
+        "lodash.memoize": "^4.1.2",
+        "make-error": "^1.3.6",
+        "semver": "^7.6.3",
+        "yargs-parser": "^21.1.1"
       },
       "dependencies": {
         "json5": {
@@ -30051,26 +30391,24 @@
           "dev": true
         },
         "semver": {
-          "version": "7.5.4",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz",
-          "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==",
-          "dev": true,
-          "requires": {
-            "lru-cache": "^6.0.0"
-          }
+          "version": "7.6.3",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
+          "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
+          "dev": true
         }
       }
     },
     "ts-loader": {
-      "version": "9.4.2",
-      "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.4.2.tgz",
-      "integrity": "sha512-OmlC4WVmFv5I0PpaxYb+qGeGOdm5giHU7HwDDUjw59emP2UYMHy9fFSDcYgSNoH8sXcj4hGCSEhlDZ9ULeDraA==",
+      "version": "9.5.1",
+      "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.5.1.tgz",
+      "integrity": "sha512-rNH3sK9kGZcH9dYzC7CewQm4NtxJTjSEVRJ2DyBZR7f8/wcta+iV44UPCXc5+nzDzivKtlzV6c9P4e+oFhDLYg==",
       "dev": true,
       "requires": {
         "chalk": "^4.1.0",
         "enhanced-resolve": "^5.0.0",
         "micromatch": "^4.0.0",
-        "semver": "^7.3.4"
+        "semver": "^7.3.4",
+        "source-map": "^0.7.4"
       },
       "dependencies": {
         "ansi-styles": {
@@ -30122,6 +30460,12 @@
             "lru-cache": "^6.0.0"
           }
         },
+        "source-map": {
+          "version": "0.7.4",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz",
+          "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==",
+          "dev": true
+        },
         "supports-color": {
           "version": "7.2.0",
           "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
@@ -30134,9 +30478,9 @@
       }
     },
     "ts-node": {
-      "version": "10.9.1",
-      "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz",
-      "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==",
+      "version": "10.9.2",
+      "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
+      "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
       "dev": true,
       "requires": {
         "@cspotcode/source-map-support": "^0.8.0",
@@ -30167,9 +30511,9 @@
       }
     },
     "tslib": {
-      "version": "2.6.3",
-      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz",
-      "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ=="
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.0.tgz",
+      "integrity": "sha512-jWVzBLplnCmoaTr13V9dYbiQ99wvZRd0vNWaDRg+aVYRcjDF3nDksxFDE/+fkXnKhpnUUkmx5pK/v8mCtLVqZA=="
     },
     "type-check": {
       "version": "0.4.0",
@@ -30249,9 +30593,9 @@
       }
     },
     "typescript": {
-      "version": "4.9.5",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
-      "integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
+      "version": "5.5.4",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz",
+      "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==",
       "dev": true
     },
     "typescript-eslint": {
@@ -30277,6 +30621,12 @@
         "which-boxed-primitive": "^1.0.2"
       }
     },
+    "undici-types": {
+      "version": "5.26.5",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+      "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+      "dev": true
+    },
     "universalify": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
@@ -30349,9 +30699,9 @@
       "dev": true
     },
     "use-sync-external-store": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz",
-      "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA=="
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.2.tgz",
+      "integrity": "sha512-PElTlVMwpblvbNqQ82d2n6RjStvdSoNe9FG28kNfz3WiXilJm4DdNkEzRhCZuIDwY8U08WVihhGR5iRqAwfDiw=="
     },
     "util-deprecate": {
       "version": "1.0.2",
@@ -30372,9 +30722,9 @@
       "dev": true
     },
     "uuid": {
-      "version": "7.0.2",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.2.tgz",
-      "integrity": "sha512-vy9V/+pKG+5ZTYKf+VcphF5Oc6EFiu3W8Nv3P3zIh0EqVI80ZxOzuPfe9EHjkFNvf8+xuTHVeei4Drydlx4zjw=="
+      "version": "7.0.3",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz",
+      "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg=="
     },
     "v8-compile-cache-lib": {
       "version": "3.0.1",
@@ -31035,9 +31385,9 @@
       "dev": true
     },
     "zrender": {
-      "version": "5.4.4",
-      "resolved": "https://registry.npmjs.org/zrender/-/zrender-5.4.4.tgz",
-      "integrity": "sha512-0VxCNJ7AGOMCWeHVyTrGzUgrK4asT4ml9PEkeGirAkKNYXYzoPJCLvmyfdoOXcjTHPs10OZVMfD1Rwg16AZyYw==",
+      "version": "5.6.0",
+      "resolved": "https://registry.npmjs.org/zrender/-/zrender-5.6.0.tgz",
+      "integrity": "sha512-uzgraf4njmmHAbEUxMJ8Oxg+P3fT04O+9p7gY+wJRVxo8Ge+KmYv0WJev945EH4wFuc4OY2NLXz46FZrWS9xJg==",
       "requires": {
         "tslib": "2.3.0"
       },
@@ -31050,11 +31400,11 @@
       }
     },
     "zustand": {
-      "version": "4.3.2",
-      "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.3.2.tgz",
-      "integrity": "sha512-rd4haDmlwMTVWVqwvgy00ny8rtti/klRoZjFbL/MAcDnmD5qSw/RZc+Vddstdv90M5Lv6RPgWvm1Hivyn0QgJw==",
+      "version": "4.5.5",
+      "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.5.tgz",
+      "integrity": "sha512-+0PALYNJNgK6hldkgDq2vLrw5f6g/jCInz52n9RTpropGgeAf/ioFUCdtsjCqu4gNhW9D01rUQBROoRjdzyn2Q==",
       "requires": {
-        "use-sync-external-store": "1.2.0"
+        "use-sync-external-store": "1.2.2"
       }
     }
   }
diff --git a/web-console/package.json b/web-console/package.json
index 106347cf13f1..96de0349e2fc 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -8,31 +8,18 @@
     "type": "git",
     "url": "https://github.com/apache/druid"
   },
-  "jest": {
-    "preset": "ts-jest",
-    "testEnvironment": "jsdom",
-    "moduleNameMapper": {
-      "\\.s?css$": "identity-obj-proxy"
-    },
-    "testMatch": [
-      "**/?(*.)+(spec).ts?(x)"
-    ],
-    "setupFilesAfterEnv": [
-      "<rootDir>src/setup-tests.ts"
-    ]
-  },
   "prettier": "@awesome-code-style/prettier-config",
   "scripts": {
     "compile": "./script/build",
     "pretest": "./script/build",
-    "jest": "jest --config jest.unit.config.js src",
+    "jest": "jest --config jest.config.js src",
     "test-base": "npm run eslint && npm run sasslint && npm run prettify-check && npm run jest",
     "test": "npm run test-base -- --silent 2>&1",
     "test-ci": "npm run test-base -- --coverage",
     "test-e2e": "jest --runInBand --config jest.e2e.config.js e2e-tests",
     "codecov": "codecov --disable=gcov -p ..",
     "coverage": "jest --coverage src",
-    "update-snapshots": "jest -u --config jest.unit.config.js",
+    "update-snapshots": "jest -u --config jest.config.js",
     "autofix": "npm run eslint-fix && npm run sasslint-fix && npm run prettify",
     "eslint": "eslint",
     "eslint-fix": "npm run eslint -- --fix",
@@ -63,72 +50,71 @@
     "not ie 11"
   ],
   "dependencies": {
-    "@blueprintjs/core": "^5.10.5",
-    "@blueprintjs/datetime": "^5.3.6",
-    "@blueprintjs/datetime2": "^2.3.7",
-    "@blueprintjs/icons": "^5.10.0",
-    "@blueprintjs/select": "^5.2.1",
-    "@druid-toolkit/query": "^0.22.22",
-    "@fontsource/open-sans": "^5.0.28",
-    "ace-builds": "~1.4.14",
-    "axios": "^1.7.4",
+    "@blueprintjs/core": "^5.13.1",
+    "@blueprintjs/datetime": "^5.3.11",
+    "@blueprintjs/datetime2": "^2.3.11",
+    "@blueprintjs/icons": "^5.13.0",
+    "@blueprintjs/select": "^5.2.5",
+    "@druid-toolkit/query": "^0.22.23",
+    "@fontsource/open-sans": "^5.0.30",
+    "ace-builds": "~1.5.3",
+    "axios": "^1.7.7",
     "chronoshift": "^0.10.0",
     "classnames": "^2.2.6",
-    "copy-to-clipboard": "^3.2.0",
-    "core-js": "^3.10.1",
-    "d3-array": "^2.12.1",
-    "d3-axis": "^2.1.0",
-    "d3-dsv": "^2.0.0",
-    "d3-scale": "^3.3.0",
+    "copy-to-clipboard": "^3.3.3",
+    "d3-array": "^3.2.4",
+    "d3-axis": "^3.0.0",
+    "d3-dsv": "^3.0.1",
+    "d3-scale": "^4.0.2",
     "d3-scale-chromatic": "^3.1.0",
-    "d3-selection": "^2.0.0",
+    "d3-selection": "^3.0.0",
     "date-fns": "^2.28.0",
-    "echarts": "^5.4.3",
+    "echarts": "^5.5.1",
     "file-saver": "^2.0.5",
     "hjson": "^3.2.2",
     "json-bigint-native": "^1.2.0",
     "lodash.debounce": "^4.0.8",
     "lodash.escape": "^4.0.1",
-    "memoize-one": "^5.1.1",
+    "memoize-one": "^5.2.1",
     "numeral": "^2.0.6",
-    "path-parse": "^1.0.7",
-    "react": "^18.2.0",
+    "react": "^18.3.1",
     "react-ace": "^10.1.0",
     "react-diff-viewer": "^3.1.1",
-    "react-dom": "^18.2.0",
+    "react-dom": "^18.3.1",
     "react-router": "^5.3.4",
     "react-router-dom": "^5.3.4",
     "react-splitter-layout": "^4.0.0",
     "react-table": "~6.11.5",
     "regenerator-runtime": "^0.13.7",
-    "tslib": "^2.3.1",
-    "uuid": "^7.0.2",
-    "zustand": "^4.3.2"
+    "tslib": "^2.8.0",
+    "uuid": "^7.0.3",
+    "zustand": "^4.5.5"
   },
   "devDependencies": {
     "@awesome-code-style/eslint-config": "^6.0.0",
     "@awesome-code-style/prettier-config": "^4.0.0",
     "@awesome-code-style/stylelint-config": "^6.0.0",
-    "@testing-library/react": "^14.0.0",
+    "@testing-library/dom": "^10.4.0",
+    "@testing-library/react": "^16.0.1",
     "@types/classnames": "^2.2.9",
-    "@types/d3-array": "^2.12.3",
-    "@types/d3-axis": "^2.1.3",
-    "@types/d3-dsv": "^2.0.0",
-    "@types/d3-scale": "^3.3.2",
+    "@types/d3-array": "^3.2.1",
+    "@types/d3-axis": "^3.0.6",
+    "@types/d3-dsv": "^3.0.7",
+    "@types/d3-scale": "^4.0.8",
     "@types/d3-scale-chromatic": "^3.0.3",
-    "@types/d3-selection": "^2.0.1",
-    "@types/enzyme": "^3.10.17",
+    "@types/d3-selection": "^3.0.11",
+    "@types/enzyme": "^3.10.18",
     "@types/enzyme-adapter-react-16": "^1.0.9",
     "@types/file-saver": "^2.0.7",
     "@types/hjson": "^2.4.6",
     "@types/jest": "^27.4.0",
-    "@types/lodash.debounce": "^4.0.6",
-    "@types/lodash.escape": "^4.0.6",
+    "@types/lodash.debounce": "^4.0.8",
+    "@types/lodash.escape": "^4.0.9",
     "@types/memoize-one": "^4.1.1",
-    "@types/node": "^12.11.7",
-    "@types/numeral": "^0.0.26",
-    "@types/react": "^18.2.42",
-    "@types/react-dom": "^18.2.17",
+    "@types/node": "^18.19.56",
+    "@types/numeral": "^2.0.5",
+    "@types/react": "^18.3.11",
+    "@types/react-dom": "^18.3.1",
     "@types/react-router-dom": "^5.3.3",
     "@types/react-splitter-layout": "^3.0.5",
     "@types/react-table": "6.8.5",
@@ -159,10 +145,10 @@
     "snarkdown": "^2.0.0",
     "style-loader": "^2.0.0",
     "stylelint": "^16.1.0",
-    "ts-jest": "^29.1.2",
-    "ts-loader": "^9.4.2",
-    "ts-node": "^10.9.1",
-    "typescript": "^4.9.5",
+    "ts-jest": "^29.2.5",
+    "ts-loader": "^9.5.1",
+    "ts-node": "^10.9.2",
+    "typescript": "~5.5.4",
     "webpack": "^5.89.0",
     "webpack-bundle-analyzer": "^4.10.1",
     "webpack-cli": "^5.1.4",
diff --git a/web-console/script/build b/web-console/script/build
index 0d83cd29fc42..ab71bf993008 100755
--- a/web-console/script/build
+++ b/web-console/script/build
@@ -19,10 +19,10 @@
 set -e
 
 echo "Adding SQL docs..."
-./script/create-sql-docs.js
+./script/create-sql-docs.mjs
 
 # add BUNDLE_ANALYZER_PLUGIN='TRUE' here to enable webpack-bundle-analyzer as a plugin
 echo "Webpacking everything..."
-NODE_ENV=production ./node_modules/.bin/webpack -c webpack.config.js --mode=production
+NODE_ENV=production ./node_modules/.bin/webpack -c webpack.config.mjs --mode=production
 
 echo "Done! Have a good day."
diff --git a/web-console/script/clean b/web-console/script/clean
index 20fb135a186b..cc5e556ec2bd 100755
--- a/web-console/script/clean
+++ b/web-console/script/clean
@@ -16,14 +16,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# remove lib/sql-function-doc.* for back compat cleanup
 rm -rf \
   coverage \
   lib/react-table.css \
-  lib/sql-docs.js \
-  lib/sql-function-doc.* \
+  lib/sql-docs.* \
   node_modules \
-  coordinator-console \
-  pages \
   public \
   index.html
diff --git a/web-console/script/create-sql-docs.js b/web-console/script/create-sql-docs.mjs
similarity index 91%
rename from web-console/script/create-sql-docs.js
rename to web-console/script/create-sql-docs.mjs
index 5df5dceb7faf..62e1f44d0177 100755
--- a/web-console/script/create-sql-docs.js
+++ b/web-console/script/create-sql-docs.mjs
@@ -18,10 +18,10 @@
  * limitations under the License.
  */
 
-const fs = require('fs-extra');
-const snarkdown = require('snarkdown');
+import fs from 'fs-extra';
+import snarkdown from 'snarkdown';
 
-const writefile = 'lib/sql-docs.js';
+const OUTPUT_FILE = 'lib/sql-docs.ts';
 
 const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 198;
 const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 15;
@@ -140,14 +140,23 @@ const readDoc = async () => {
 // This file is auto generated and should not be modified
 
 // prettier-ignore
-exports.SQL_DATA_TYPES = ${JSON.stringify(dataTypeDocs, null, 2)};
+export const SQL_DATA_TYPES: Record<string, [runtime: string, description: string]> = ${JSON.stringify(
+    dataTypeDocs,
+    null,
+    2,
+  )};
 
 // prettier-ignore
-exports.SQL_FUNCTIONS = ${JSON.stringify(functionDocs, null, 2)};
+export const SQL_FUNCTIONS: Record<string, [args: string, description: string][]> = ${JSON.stringify(
+    functionDocs,
+    null,
+    2,
+  )};
 `;
 
+  // eslint-disable-next-line no-undef
   console.log(`Found ${numDataTypes} data types and ${numFunction} functions`);
-  await fs.writeFile(writefile, content, 'utf-8');
+  await fs.writeFile(OUTPUT_FILE, content, 'utf-8');
 };
 
 readDoc();
diff --git a/web-console/src/ace-modes/dsql.ts b/web-console/src/ace-modes/dsql.ts
index 38cc40124e2f..0c4daac5bfbf 100644
--- a/web-console/src/ace-modes/dsql.ts
+++ b/web-console/src/ace-modes/dsql.ts
@@ -24,8 +24,13 @@
 import type { Ace } from 'ace-builds';
 import ace from 'ace-builds/src-noconflict/ace';
 
-import * as druidKeywords from '../../lib/keywords';
-import * as druidFunctions from '../../lib/sql-docs';
+import {
+  SQL_CONSTANTS,
+  SQL_DYNAMICS,
+  SQL_EXPRESSION_PARTS,
+  SQL_KEYWORDS,
+} from '../../lib/keywords';
+import { SQL_DATA_TYPES, SQL_FUNCTIONS } from '../../lib/sql-docs';
 
 import type { ItemDescription } from './make-doc-html';
 import { makeDocHtml } from './make-doc-html';
@@ -41,20 +46,16 @@ ace.define(
 
     const SqlHighlightRules = function (this: any) {
       // Stuff like: 'with|select|from|where|and|or|group|by|order|limit|having|as|case|'
-      const keywords = druidKeywords.SQL_KEYWORDS.concat(druidKeywords.SQL_EXPRESSION_PARTS)
-        .join('|')
-        .replace(/\s/g, '|');
+      const keywords = SQL_KEYWORDS.concat(SQL_EXPRESSION_PARTS).join('|').replace(/\s/g, '|');
 
       // Stuff like: 'true|false'
-      const builtinConstants = druidKeywords.SQL_CONSTANTS.join('|');
+      const builtinConstants = SQL_CONSTANTS.join('|');
 
       // Stuff like: 'avg|count|first|last|max|min'
-      const builtinFunctions = druidKeywords.SQL_DYNAMICS.concat(
-        Object.keys(druidFunctions.SQL_FUNCTIONS),
-      ).join('|');
+      const builtinFunctions = SQL_DYNAMICS.concat(Object.keys(SQL_FUNCTIONS)).join('|');
 
       // Stuff like: 'int|numeric|decimal|date|varchar|char|bigint|float|double|bit|binary|text|set|timestamp'
-      const dataTypes = Object.keys(druidFunctions.SQL_DATA_TYPES).join('|');
+      const dataTypes = Object.keys(SQL_DATA_TYPES).join('|');
 
       const keywordMapper = this.createKeywordMapper(
         {
@@ -136,16 +137,16 @@ ace.define(
     const SqlHighlightRules = acequire('./dsql_highlight_rules').SqlHighlightRules;
 
     const completions = ([] as Ace.Completion[]).concat(
-      druidKeywords.SQL_KEYWORDS.map(v => ({ name: v, value: v, score: 0, meta: 'keyword' })),
-      druidKeywords.SQL_EXPRESSION_PARTS.map(v => ({
+      SQL_KEYWORDS.map(v => ({ name: v, value: v, score: 0, meta: 'keyword' })),
+      SQL_EXPRESSION_PARTS.map(v => ({
         name: v,
         value: v,
         score: 0,
         meta: 'keyword',
       })),
-      druidKeywords.SQL_CONSTANTS.map(v => ({ name: v, value: v, score: 0, meta: 'constant' })),
-      druidKeywords.SQL_DYNAMICS.map(v => ({ name: v, value: v, score: 0, meta: 'dynamic' })),
-      Object.entries(druidFunctions.SQL_DATA_TYPES).map(([name, [runtime, description]]) => {
+      SQL_CONSTANTS.map(v => ({ name: v, value: v, score: 0, meta: 'constant' })),
+      SQL_DYNAMICS.map(v => ({ name: v, value: v, score: 0, meta: 'dynamic' })),
+      Object.entries(SQL_DATA_TYPES).map(([name, [runtime, description]]) => {
         const item: ItemDescription = {
           name,
           description,
@@ -160,7 +161,7 @@ ace.define(
           docText: description,
         };
       }),
-      Object.entries(druidFunctions.SQL_FUNCTIONS).flatMap(([name, versions]) => {
+      Object.entries(SQL_FUNCTIONS).flatMap(([name, versions]) => {
         return versions.map(([args, description]) => {
           const item = { name, description, syntax: `${name}(${args})` };
           return {
diff --git a/web-console/src/components/menu-boolean/__snapshots__/menu-boolean.spec.tsx.snap b/web-console/src/components/menu-boolean/__snapshots__/menu-boolean.spec.tsx.snap
index 3fc368ecd355..e443845c03f5 100644
--- a/web-console/src/components/menu-boolean/__snapshots__/menu-boolean.spec.tsx.snap
+++ b/web-console/src/components/menu-boolean/__snapshots__/menu-boolean.spec.tsx.snap
@@ -11,7 +11,6 @@ exports[`MenuBoolean matches snapshot false 1`] = `
     tabindex="0"
   >
     <a
-      aria-expanded="false"
       aria-haspopup="menu"
       class="bp5-menu-item menu-tristate"
       label="false"
@@ -65,7 +64,6 @@ exports[`MenuBoolean matches snapshot no undefined 1`] = `
     tabindex="0"
   >
     <a
-      aria-expanded="false"
       aria-haspopup="menu"
       class="bp5-menu-item menu-tristate"
       label="false"
@@ -119,7 +117,6 @@ exports[`MenuBoolean matches snapshot undefined 1`] = `
     tabindex="0"
   >
     <a
-      aria-expanded="false"
       aria-haspopup="menu"
       class="bp5-menu-item menu-tristate"
       label="Auto (true)"
diff --git a/web-console/src/components/segment-timeline/__snapshots__/segment-timeline.spec.tsx.snap b/web-console/src/components/segment-timeline/__snapshots__/segment-timeline.spec.tsx.snap
index 1d88f973f712..ae30df246c2b 100644
--- a/web-console/src/components/segment-timeline/__snapshots__/segment-timeline.spec.tsx.snap
+++ b/web-console/src/components/segment-timeline/__snapshots__/segment-timeline.spec.tsx.snap
@@ -59,9 +59,13 @@ exports[`SegmentTimeline matches snapshot 1`] = `
       >
         <div
           class="bp5-segmented-control bp5-fill"
+          role="radiogroup"
         >
           <button
+            aria-checked="true"
             class="bp5-button"
+            role="radio"
+            tabindex="0"
             type="button"
           >
             <span
@@ -71,7 +75,10 @@ exports[`SegmentTimeline matches snapshot 1`] = `
             </span>
           </button>
           <button
+            aria-checked="false"
             class="bp5-button bp5-minimal"
+            role="radio"
+            tabindex="-1"
             type="button"
           >
             <span
diff --git a/web-console/src/components/suggestion-menu/__snapshots__/suggestion-menu.spec.tsx.snap b/web-console/src/components/suggestion-menu/__snapshots__/suggestion-menu.spec.tsx.snap
index 67ca637b18d0..c3c275a6312c 100644
--- a/web-console/src/components/suggestion-menu/__snapshots__/suggestion-menu.spec.tsx.snap
+++ b/web-console/src/components/suggestion-menu/__snapshots__/suggestion-menu.spec.tsx.snap
@@ -63,7 +63,6 @@ exports[`SuggestionMenu matches snapshot 1`] = `
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
diff --git a/web-console/src/entry.tsx b/web-console/src/entry.tsx
index 2faca26b22eb..42cfe08ad13c 100644
--- a/web-console/src/entry.tsx
+++ b/web-console/src/entry.tsx
@@ -16,7 +16,6 @@
  * limitations under the License.
  */
 
-import 'core-js/stable';
 import 'regenerator-runtime/runtime';
 import './bootstrap/ace';
 
diff --git a/web-console/src/setup-tests.ts b/web-console/src/setup-tests.ts
index aa6cd6753b2f..0a189c543570 100644
--- a/web-console/src/setup-tests.ts
+++ b/web-console/src/setup-tests.ts
@@ -16,7 +16,6 @@
  * limitations under the License.
  */
 
-import 'core-js/stable';
 import './bootstrap/ace';
 
 import { Icons } from '@blueprintjs/icons';
diff --git a/web-console/src/views/explore-view/utils/table-query.ts b/web-console/src/views/explore-view/utils/table-query.ts
index d66eca0c7d7e..bc5b6484c9a3 100644
--- a/web-console/src/views/explore-view/utils/table-query.ts
+++ b/web-console/src/views/explore-view/utils/table-query.ts
@@ -668,7 +668,7 @@ function makeJoinCompareTableQueryAndHints(
           : q
               .applyIf(orderByCompareDuration, q =>
                 q.changeOrderByExpression(
-                  decodedOrderBy!.orderedMeasure!.expression.toOrderByExpression('DESC'),
+                  decodedOrderBy.orderedMeasure!.expression.toOrderByExpression('DESC'),
                 ),
               )
               .changeLimitValue(safeSubQueryLimit),
@@ -701,7 +701,7 @@ function makeJoinCompareTableQueryAndHints(
               q =>
                 q
                   .changeOrderByExpression(
-                    effectiveOrderBy.changeExpression(C(decodedOrderBy!.orderedMeasure!.name)),
+                    effectiveOrderBy.changeExpression(C(decodedOrderBy.orderedMeasure!.name)),
                   )
                   .changeLimitValue(maxRows),
               q =>
@@ -716,7 +716,7 @@ function makeJoinCompareTableQueryAndHints(
                     )
                   : q
                       .changeOrderByExpression(
-                        C(decodedOrderBy!.orderedThing.name).toOrderByExpression('DESC'),
+                        C(decodedOrderBy.orderedThing.name).toOrderByExpression('DESC'),
                       )
                       .changeLimitValue(safeSubQueryLimit),
             ),
diff --git a/web-console/src/views/workbench-view/column-tree/column-tree-menu/complex-menu-items/__snapshots__/complex-menu-items.spec.tsx.snap b/web-console/src/views/workbench-view/column-tree/column-tree-menu/complex-menu-items/__snapshots__/complex-menu-items.spec.tsx.snap
index b2106bcbe527..803dffaa47bb 100644
--- a/web-console/src/views/workbench-view/column-tree/column-tree-menu/complex-menu-items/__snapshots__/complex-menu-items.spec.tsx.snap
+++ b/web-console/src/views/workbench-view/column-tree/column-tree-menu/complex-menu-items/__snapshots__/complex-menu-items.spec.tsx.snap
@@ -12,7 +12,6 @@ exports[`ComplexMenuItems matches snapshot when menu is opened for column not in
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
diff --git a/web-console/src/views/workbench-view/column-tree/column-tree-menu/number-menu-items/__snapshots__/number-menu-items.spec.tsx.snap b/web-console/src/views/workbench-view/column-tree/column-tree-menu/number-menu-items/__snapshots__/number-menu-items.spec.tsx.snap
index a5c19a2b8901..f09e20ee90c0 100644
--- a/web-console/src/views/workbench-view/column-tree/column-tree-menu/number-menu-items/__snapshots__/number-menu-items.spec.tsx.snap
+++ b/web-console/src/views/workbench-view/column-tree/column-tree-menu/number-menu-items/__snapshots__/number-menu-items.spec.tsx.snap
@@ -12,7 +12,6 @@ exports[`NumberMenuItems matches snapshot when menu is opened for column inside
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -79,7 +78,6 @@ exports[`NumberMenuItems matches snapshot when menu is opened for column inside
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -184,7 +182,6 @@ exports[`NumberMenuItems matches snapshot when menu is opened for column inside
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -256,7 +253,6 @@ exports[`NumberMenuItems matches snapshot when menu is opened for column not ins
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -323,7 +319,6 @@ exports[`NumberMenuItems matches snapshot when menu is opened for column not ins
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -390,7 +385,6 @@ exports[`NumberMenuItems matches snapshot when menu is opened for column not ins
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
diff --git a/web-console/src/views/workbench-view/column-tree/column-tree-menu/string-menu-items/__snapshots__/string-menu-items.spec.tsx.snap b/web-console/src/views/workbench-view/column-tree/column-tree-menu/string-menu-items/__snapshots__/string-menu-items.spec.tsx.snap
index 034c72eef606..f7ed0276efc1 100644
--- a/web-console/src/views/workbench-view/column-tree/column-tree-menu/string-menu-items/__snapshots__/string-menu-items.spec.tsx.snap
+++ b/web-console/src/views/workbench-view/column-tree/column-tree-menu/string-menu-items/__snapshots__/string-menu-items.spec.tsx.snap
@@ -12,7 +12,6 @@ exports[`StringMenuItems matches snapshot when menu is opened for column inside
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -79,7 +78,6 @@ exports[`StringMenuItems matches snapshot when menu is opened for column inside
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -184,7 +182,6 @@ exports[`StringMenuItems matches snapshot when menu is opened for column inside
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -256,7 +253,6 @@ exports[`StringMenuItems matches snapshot when menu is opened for column not ins
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -323,7 +319,6 @@ exports[`StringMenuItems matches snapshot when menu is opened for column not ins
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -390,7 +385,6 @@ exports[`StringMenuItems matches snapshot when menu is opened for column not ins
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
diff --git a/web-console/src/views/workbench-view/column-tree/column-tree-menu/time-menu-items/__snapshots__/time-menu-items.spec.tsx.snap b/web-console/src/views/workbench-view/column-tree/column-tree-menu/time-menu-items/__snapshots__/time-menu-items.spec.tsx.snap
index 8b6ec0a85df8..6ec9e65d06d3 100644
--- a/web-console/src/views/workbench-view/column-tree/column-tree-menu/time-menu-items/__snapshots__/time-menu-items.spec.tsx.snap
+++ b/web-console/src/views/workbench-view/column-tree/column-tree-menu/time-menu-items/__snapshots__/time-menu-items.spec.tsx.snap
@@ -12,7 +12,6 @@ exports[`TimeMenuItems matches snapshot when menu is opened for column inside gr
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -79,7 +78,6 @@ exports[`TimeMenuItems matches snapshot when menu is opened for column inside gr
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -184,7 +182,6 @@ exports[`TimeMenuItems matches snapshot when menu is opened for column inside gr
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -256,7 +253,6 @@ exports[`TimeMenuItems matches snapshot when menu is opened for column not insid
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -323,7 +319,6 @@ exports[`TimeMenuItems matches snapshot when menu is opened for column not insid
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
@@ -390,7 +385,6 @@ exports[`TimeMenuItems matches snapshot when menu is opened for column not insid
       tabindex="0"
     >
       <a
-        aria-expanded="false"
         aria-haspopup="menu"
         class="bp5-menu-item"
         role="none"
diff --git a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx
index cb4e9e3b192e..a77ed004ea26 100644
--- a/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx
+++ b/web-console/src/views/workbench-view/flexible-query-input/flexible-query-input.tsx
@@ -355,6 +355,7 @@ export class FlexibleQueryInput extends React.PureComponent<
                   ),
                   'sub-query-highlight',
                   'text',
+                  false,
                 );
               this.highlightFoundQuery = { row, marker };
             }}
diff --git a/web-console/src/views/workbench-view/run-panel/run-panel.spec.tsx b/web-console/src/views/workbench-view/run-panel/run-panel.spec.tsx
index af04fd324c1f..64fcb98b7744 100644
--- a/web-console/src/views/workbench-view/run-panel/run-panel.spec.tsx
+++ b/web-console/src/views/workbench-view/run-panel/run-panel.spec.tsx
@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+import { HotkeysProvider } from '@blueprintjs/core';
 import { render } from '@testing-library/react';
 
 import { DEFAULT_SERVER_QUERY_CONTEXT, DRUID_ENGINES, WorkbenchQuery } from '../../../druid-models';
@@ -25,15 +26,17 @@ import { RunPanel } from './run-panel';
 describe('RunPanel', () => {
   it('matches snapshot on native (auto) query', () => {
     const runPanel = (
-      <RunPanel
-        query={WorkbenchQuery.blank().changeQueryString(`SELECT * FROM wikipedia`)}
-        onQueryChange={() => {}}
-        running={false}
-        onRun={() => {}}
-        queryEngines={DRUID_ENGINES}
-        clusterCapacity={9}
-        defaultQueryContext={DEFAULT_SERVER_QUERY_CONTEXT}
-      />
+      <HotkeysProvider>
+        <RunPanel
+          query={WorkbenchQuery.blank().changeQueryString(`SELECT * FROM wikipedia`)}
+          onQueryChange={() => {}}
+          running={false}
+          onRun={() => {}}
+          queryEngines={DRUID_ENGINES}
+          clusterCapacity={9}
+          defaultQueryContext={DEFAULT_SERVER_QUERY_CONTEXT}
+        />
+      </HotkeysProvider>
     );
     const { container } = render(runPanel);
     expect(container.firstChild).toMatchSnapshot();
@@ -41,17 +44,19 @@ describe('RunPanel', () => {
 
   it('matches snapshot on msq (auto) query', () => {
     const runPanel = (
-      <RunPanel
-        query={WorkbenchQuery.blank()
-          .changeQueryString(`SELECT * FROM wikipedia`)
-          .changeEngine('sql-msq-task')}
-        onQueryChange={() => {}}
-        running={false}
-        onRun={() => {}}
-        queryEngines={DRUID_ENGINES}
-        clusterCapacity={9}
-        defaultQueryContext={DEFAULT_SERVER_QUERY_CONTEXT}
-      />
+      <HotkeysProvider>
+        <RunPanel
+          query={WorkbenchQuery.blank()
+            .changeQueryString(`SELECT * FROM wikipedia`)
+            .changeEngine('sql-msq-task')}
+          onQueryChange={() => {}}
+          running={false}
+          onRun={() => {}}
+          queryEngines={DRUID_ENGINES}
+          clusterCapacity={9}
+          defaultQueryContext={DEFAULT_SERVER_QUERY_CONTEXT}
+        />
+      </HotkeysProvider>
     );
     const { container } = render(runPanel);
     expect(container.firstChild).toMatchSnapshot();
diff --git a/web-console/tsconfig.test.json b/web-console/tsconfig.test.json
index 2c7b284162f4..7917cd416e8f 100644
--- a/web-console/tsconfig.test.json
+++ b/web-console/tsconfig.test.json
@@ -1,6 +1,7 @@
 {
   "extends": "./tsconfig.json",
   "compilerOptions": {
+    "allowJs": true,
     "module": "commonjs"
   }
 }
diff --git a/web-console/webpack.config.js b/web-console/webpack.config.mjs
similarity index 88%
rename from web-console/webpack.config.js
rename to web-console/webpack.config.mjs
index 637a3d7569a5..1390444cd388 100644
--- a/web-console/webpack.config.js
+++ b/web-console/webpack.config.mjs
@@ -16,13 +16,15 @@
  * limitations under the License.
  */
 
-const process = require('process');
-const path = require('path');
-const { SassString } = require('sass');
-const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
-const webpack = require('webpack');
+import fs from 'fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, resolve } from 'path';
+import process from 'process';
+import sass from 'sass';
+import webpack from 'webpack';
+import { BundleAnalyzerPlugin } from 'webpack-bundle-analyzer';
 
-const { version } = require('./package.json');
+const version = JSON.parse(fs.readFileSync('./package.json', 'utf-8')).version;
 
 const supportedLocales = ['en-US'];
 
@@ -30,7 +32,13 @@ function friendlyErrorFormatter(e) {
   return `${e.severity}: ${e.content} [TS${e.code}]\n    at (${e.file}:${e.line}:${e.character})`;
 }
 
-module.exports = env => {
+// Get the file path of the current module
+const __filename = fileURLToPath(import.meta.url);
+
+// Get the directory name of the current module
+const __dirname = dirname(__filename);
+
+export default env => {
   let druidUrl = (env || {}).druid_host || process.env.druid_host || 'localhost';
   if (!druidUrl.startsWith('http')) {
     druidUrl = (druidUrl.endsWith(':9088') ? 'https://' : 'http://') + druidUrl;
@@ -45,6 +53,8 @@ module.exports = env => {
   };
 
   const mode = process.env.NODE_ENV === 'production' ? 'production' : 'development';
+
+  // eslint-disable-next-line no-undef
   console.log(`Webpack running in ${mode} mode.`);
 
   const plugins = [
@@ -69,7 +79,7 @@ module.exports = env => {
       'web-console': './src/entry.tsx',
     },
     output: {
-      path: path.resolve(__dirname, './public'),
+      path: resolve(__dirname, './public'),
       filename: `[name]-${version}.js`,
       chunkFilename: `[name]-${version}.js`,
       publicPath: 'public/',
@@ -154,7 +164,7 @@ module.exports = env => {
                     // via CSS are themselves being used by the web console, so we can safely omit the icons.
                     //
                     // TODO: Re-evaluate after upgrading to Blueprint v6
-                    'svg-icon($_icon, $_path)': () => new SassString('transparent'),
+                    'svg-icon($_icon, $_path)': () => new sass.SassString('transparent'),
                   },
                 },
               },

From 9dfb37871162d884505ac4f012270502644adeb6 Mon Sep 17 00:00:00 2001
From: Kashif Faraz <kashif.faraz@gmail.com>
Date: Tue, 22 Oct 2024 09:03:46 +0530
Subject: [PATCH 05/12] Remove unused coordinator dynamic configs
 mergeSegmentsLimit, mergeBytesLimit (#17384)

* Remove unused coordinator dynamic configs

* Update docs and web-console
---
 .../dynamic-configuration-api.md              |  8 +-
 docs/configuration/index.md                   |  2 -
 .../coordinator/CoordinatorDynamicConfig.java | 38 +-------
 .../http/CoordinatorDynamicConfigTest.java    | 88 +------------------
 .../coordinator-dynamic-config.mock.ts        |  2 -
 .../coordinator-dynamic-config.tsx            | 14 ---
 6 files changed, 3 insertions(+), 149 deletions(-)

diff --git a/docs/api-reference/dynamic-configuration-api.md b/docs/api-reference/dynamic-configuration-api.md
index 85c7dcf5e492..90b7028f2478 100644
--- a/docs/api-reference/dynamic-configuration-api.md
+++ b/docs/api-reference/dynamic-configuration-api.md
@@ -91,8 +91,6 @@ Host: http://ROUTER_IP:ROUTER_PORT
 ```json
 {
     "millisToWaitBeforeDeleting": 900000,
-    "mergeBytesLimit": 524288000,
-    "mergeSegmentsLimit": 100,
     "maxSegmentsToMove": 100,
     "replicantLifetime": 15,
     "replicationThrottleLimit": 500,
@@ -159,8 +157,6 @@ curl "http://ROUTER_IP:ROUTER_PORT/druid/coordinator/v1/config" \
 --header 'Content-Type: application/json' \
 --data '{
   "millisToWaitBeforeDeleting": 900000,
-  "mergeBytesLimit": 524288000,
-  "mergeSegmentsLimit": 100,
   "maxSegmentsToMove": 5,
   "percentOfSegmentsToConsiderPerMove": 100,
   "useBatchedSegmentSampler": true,
@@ -192,8 +188,6 @@ Content-Length: 683
 
 {
   "millisToWaitBeforeDeleting": 900000,
-  "mergeBytesLimit": 524288000,
-  "mergeSegmentsLimit": 100,
   "maxSegmentsToMove": 5,
   "percentOfSegmentsToConsiderPerMove": 100,
   "useBatchedSegmentSampler": true,
@@ -295,7 +289,7 @@ Host: http://ROUTER_IP:ROUTER_PORT
             "comment": "",
             "ip": "127.0.0.1"
         },
-        "payload": "{\"millisToWaitBeforeDeleting\":900000,\"mergeBytesLimit\":524288000,\"mergeSegmentsLimit\":100,\"maxSegmentsToMove\":5,\"replicantLifetime\":15,\"replicationThrottleLimit\":10,\"balancerComputeThreads\":1,\"killDataSourceWhitelist\":[],\"killPendingSegmentsSkipList\":[],\"maxSegmentsInNodeLoadingQueue\":100,\"decommissioningNodes\":[],\"decommissioningMaxPercentOfMaxSegmentsToMove\":70,\"pauseCoordination\":false,\"replicateAfterLoadTimeout\":false,\"maxNonPrimaryReplicantsToLoad\":2147483647,\"useRoundRobinSegmentAssignment\":true,\"smartSegmentLoading\":true,\"debugDimensions\":null}",
+        "payload": "{\"millisToWaitBeforeDeleting\":900000,\"maxSegmentsToMove\":5,\"replicantLifetime\":15,\"replicationThrottleLimit\":10,\"balancerComputeThreads\":1,\"killDataSourceWhitelist\":[],\"killPendingSegmentsSkipList\":[],\"maxSegmentsInNodeLoadingQueue\":100,\"decommissioningNodes\":[],\"decommissioningMaxPercentOfMaxSegmentsToMove\":70,\"pauseCoordination\":false,\"replicateAfterLoadTimeout\":false,\"maxNonPrimaryReplicantsToLoad\":2147483647,\"useRoundRobinSegmentAssignment\":true,\"smartSegmentLoading\":true,\"debugDimensions\":null}",
         "auditTime": "2023-10-03T20:59:51.622Z"
     }
 ]
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index a8dd77efc01a..8f4f8065f1d9 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -951,8 +951,6 @@ The following table shows the dynamic configuration properties for the Coordinat
 |Property|Description|Default|
 |--------|-----------|-------|
 |`millisToWaitBeforeDeleting`|How long does the Coordinator need to be a leader before it can start marking overshadowed segments as unused in metadata storage.| 900000 (15 mins)|
-|`mergeBytesLimit`|The maximum total uncompressed size in bytes of segments to merge.|524288000L|
-|`mergeSegmentsLimit`|The maximum number of segments that can be in a single [append task](../ingestion/tasks.md).|100|
 |`smartSegmentLoading`|Enables ["smart" segment loading mode](#smart-segment-loading) which dynamically computes the optimal values of several properties that maximize Coordinator performance.|true|
 |`maxSegmentsToMove`|The maximum number of segments that can be moved in a Historical tier at any given time.|100|
 |`replicantLifetime`|The maximum number of Coordinator runs for which a segment can wait in the load queue of a Historical before Druid raises an alert.|15|
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java
index e834c7b1fd33..d805bad5e01c 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java
@@ -51,8 +51,6 @@ public class CoordinatorDynamicConfig
   public static final String CONFIG_KEY = "coordinator.config";
 
   private final long markSegmentAsUnusedDelayMillis;
-  private final long mergeBytesLimit;
-  private final int mergeSegmentsLimit;
   private final int maxSegmentsToMove;
   private final int replicantLifetime;
   private final int replicationThrottleLimit;
@@ -98,8 +96,6 @@ public CoordinatorDynamicConfig(
       // updated to Jackson 2.9 it could be changed, see https://github.com/apache/druid/issues/7152
       @JsonProperty("millisToWaitBeforeDeleting")
           long markSegmentAsUnusedDelayMillis,
-      @JsonProperty("mergeBytesLimit") long mergeBytesLimit,
-      @JsonProperty("mergeSegmentsLimit") int mergeSegmentsLimit,
       @JsonProperty("maxSegmentsToMove") int maxSegmentsToMove,
       @JsonProperty("replicantLifetime") int replicantLifetime,
       @JsonProperty("replicationThrottleLimit") int replicationThrottleLimit,
@@ -127,8 +123,6 @@ public CoordinatorDynamicConfig(
   {
     this.markSegmentAsUnusedDelayMillis =
         markSegmentAsUnusedDelayMillis;
-    this.mergeBytesLimit = mergeBytesLimit;
-    this.mergeSegmentsLimit = mergeSegmentsLimit;
     this.maxSegmentsToMove = maxSegmentsToMove;
     this.smartSegmentLoading = Builder.valueOrDefault(smartSegmentLoading, Defaults.SMART_SEGMENT_LOADING);
 
@@ -212,18 +206,6 @@ public long getMarkSegmentAsUnusedDelayMillis()
     return markSegmentAsUnusedDelayMillis;
   }
 
-  @JsonProperty
-  public long getMergeBytesLimit()
-  {
-    return mergeBytesLimit;
-  }
-
-  @JsonProperty
-  public int getMergeSegmentsLimit()
-  {
-    return mergeSegmentsLimit;
-  }
-
   @JsonProperty
   public int getMaxSegmentsToMove()
   {
@@ -294,7 +276,7 @@ public boolean isSmartSegmentLoading()
    * List of historical servers to 'decommission'. Coordinator will not assign new segments to 'decommissioning'
    * servers, and segments will be moved away from them to be placed on non-decommissioning servers.
    *
-   * @return list of host:port entries
+   * @return Set of host:port entries
    */
   @JsonProperty
   public Set<String> getDecommissioningNodes()
@@ -332,8 +314,6 @@ public String toString()
     return "CoordinatorDynamicConfig{" +
            "leadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments="
            + markSegmentAsUnusedDelayMillis +
-           ", mergeBytesLimit=" + mergeBytesLimit +
-           ", mergeSegmentsLimit=" + mergeSegmentsLimit +
            ", maxSegmentsToMove=" + maxSegmentsToMove +
            ", replicantLifetime=" + replicantLifetime +
            ", replicationThrottleLimit=" + replicationThrottleLimit +
@@ -362,8 +342,6 @@ public boolean equals(Object o)
     CoordinatorDynamicConfig that = (CoordinatorDynamicConfig) o;
 
     return markSegmentAsUnusedDelayMillis == that.markSegmentAsUnusedDelayMillis
-           && mergeBytesLimit == that.mergeBytesLimit
-           && mergeSegmentsLimit == that.mergeSegmentsLimit
            && maxSegmentsToMove == that.maxSegmentsToMove
            && balancerComputeThreads == that.balancerComputeThreads
            && replicantLifetime == that.replicantLifetime
@@ -389,8 +367,6 @@ public int hashCode()
   {
     return Objects.hash(
         markSegmentAsUnusedDelayMillis,
-        mergeBytesLimit,
-        mergeSegmentsLimit,
         maxSegmentsToMove,
         replicantLifetime,
         replicationThrottleLimit,
@@ -423,8 +399,6 @@ public static int getDefaultBalancerComputeThreads()
   private static class Defaults
   {
     static final long LEADING_MILLIS_BEFORE_MARK_UNUSED = TimeUnit.MINUTES.toMillis(15);
-    static final long MERGE_BYTES_LIMIT = 524_288_000L;
-    static final int MERGE_SEGMENTS_LIMIT = 100;
     static final int MAX_SEGMENTS_TO_MOVE = 100;
     static final int REPLICANT_LIFETIME = 15;
     static final int REPLICATION_THROTTLE_LIMIT = 500;
@@ -441,8 +415,6 @@ private static class Defaults
   public static class Builder
   {
     private Long markSegmentAsUnusedDelayMillis;
-    private Long mergeBytesLimit;
-    private Integer mergeSegmentsLimit;
     private Integer maxSegmentsToMove;
     private Integer replicantLifetime;
     private Integer replicationThrottleLimit;
@@ -466,8 +438,6 @@ public Builder()
     @JsonCreator
     public Builder(
         @JsonProperty("millisToWaitBeforeDeleting") @Nullable Long markSegmentAsUnusedDelayMillis,
-        @JsonProperty("mergeBytesLimit") @Nullable Long mergeBytesLimit,
-        @JsonProperty("mergeSegmentsLimit") @Nullable Integer mergeSegmentsLimit,
         @JsonProperty("maxSegmentsToMove") @Nullable Integer maxSegmentsToMove,
         @JsonProperty("replicantLifetime") @Nullable Integer replicantLifetime,
         @JsonProperty("replicationThrottleLimit") @Nullable Integer replicationThrottleLimit,
@@ -486,8 +456,6 @@ public Builder(
     )
     {
       this.markSegmentAsUnusedDelayMillis = markSegmentAsUnusedDelayMillis;
-      this.mergeBytesLimit = mergeBytesLimit;
-      this.mergeSegmentsLimit = mergeSegmentsLimit;
       this.maxSegmentsToMove = maxSegmentsToMove;
       this.replicantLifetime = replicantLifetime;
       this.replicationThrottleLimit = replicationThrottleLimit;
@@ -600,8 +568,6 @@ public CoordinatorDynamicConfig build()
               markSegmentAsUnusedDelayMillis,
               Defaults.LEADING_MILLIS_BEFORE_MARK_UNUSED
           ),
-          valueOrDefault(mergeBytesLimit, Defaults.MERGE_BYTES_LIMIT),
-          valueOrDefault(mergeSegmentsLimit, Defaults.MERGE_SEGMENTS_LIMIT),
           valueOrDefault(maxSegmentsToMove, Defaults.MAX_SEGMENTS_TO_MOVE),
           valueOrDefault(replicantLifetime, Defaults.REPLICANT_LIFETIME),
           valueOrDefault(replicationThrottleLimit, Defaults.REPLICATION_THROTTLE_LIMIT),
@@ -632,8 +598,6 @@ public CoordinatorDynamicConfig build(CoordinatorDynamicConfig defaults)
               markSegmentAsUnusedDelayMillis,
               defaults.getMarkSegmentAsUnusedDelayMillis()
           ),
-          valueOrDefault(mergeBytesLimit, defaults.getMergeBytesLimit()),
-          valueOrDefault(mergeSegmentsLimit, defaults.getMergeSegmentsLimit()),
           valueOrDefault(maxSegmentsToMove, defaults.getMaxSegmentsToMove()),
           valueOrDefault(replicantLifetime, defaults.getReplicantLifetime()),
           valueOrDefault(replicationThrottleLimit, defaults.getReplicationThrottleLimit()),
diff --git a/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java
index a97160ed27be..eb7fb1992873 100644
--- a/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java
+++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java
@@ -41,8 +41,6 @@ public void testSerde() throws Exception
   {
     String jsonStr = "{\n"
                      + "  \"millisToWaitBeforeDeleting\": 1,\n"
-                     + "  \"mergeBytesLimit\": 1,\n"
-                     + "  \"mergeSegmentsLimit\" : 1,\n"
                      + "  \"maxSegmentsToMove\": 1,\n"
                      + "  \"replicantLifetime\": 1,\n"
                      + "  \"replicationThrottleLimit\": 1,\n"
@@ -73,13 +71,10 @@ public void testSerde() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.15,
         2,
-        false,
         1,
         decommissioning,
         false,
@@ -93,13 +88,10 @@ public void testSerde() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.15,
         2,
-        false,
         1,
         ImmutableSet.of("host1"),
         false,
@@ -113,13 +105,10 @@ public void testSerde() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.15,
         2,
-        false,
         1,
         ImmutableSet.of("host1"),
         false,
@@ -133,13 +122,10 @@ public void testSerde() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.15,
         2,
-        false,
         1,
         ImmutableSet.of("host1"),
         true,
@@ -153,13 +139,10 @@ public void testSerde() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.15,
         2,
-        false,
         1,
         ImmutableSet.of("host1"),
         true,
@@ -173,13 +156,10 @@ public void testSerde() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.15,
         2,
-        false,
         1,
         ImmutableSet.of("host1"),
         true,
@@ -193,13 +173,10 @@ public void testSerde() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.1,
         2,
-        false,
         1,
         ImmutableSet.of("host1"),
         true,
@@ -213,13 +190,10 @@ public void testSerde() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.1,
         5,
-        false,
         1,
         ImmutableSet.of("host1"),
         true,
@@ -244,8 +218,6 @@ public void testDeserializationWithUnknownProperties() throws Exception
   public void testConstructorWithNullsShouldKillUnusedSegmentsInAllDataSources()
   {
     CoordinatorDynamicConfig config = new CoordinatorDynamicConfig(
-        1,
-        1,
         1,
         1,
         1,
@@ -270,8 +242,6 @@ public void testConstructorWithNullsShouldKillUnusedSegmentsInAllDataSources()
   public void testConstructorWithSpecificDataSourcesToKillShouldNotKillUnusedSegmentsInAllDatasources()
   {
     CoordinatorDynamicConfig config = new CoordinatorDynamicConfig(
-        1,
-        1,
         1,
         1,
         1,
@@ -297,8 +267,6 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio
   {
     String jsonStr = "{\n"
                      + "  \"millisToWaitBeforeDeleting\": 1,\n"
-                     + "  \"mergeBytesLimit\": 1,\n"
-                     + "  \"mergeSegmentsLimit\" : 1,\n"
                      + "  \"maxSegmentsToMove\": 1,\n"
                      + "  \"replicantLifetime\": 1,\n"
                      + "  \"replicationThrottleLimit\": 1,\n"
@@ -324,13 +292,10 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.1,
         Integer.MAX_VALUE,
-        false,
         1,
         decommissioning,
         false,
@@ -344,13 +309,10 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.1,
         Integer.MAX_VALUE,
-        false,
         1,
         ImmutableSet.of("host1"),
         false,
@@ -364,13 +326,10 @@ public void testDecommissioningParametersBackwardCompatibility() throws Exceptio
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.1,
         Integer.MAX_VALUE,
-        false,
         1,
         ImmutableSet.of("host1"),
         false,
@@ -383,8 +342,6 @@ public void testSerdeWithStringInKillDataSourceWhitelist() throws Exception
   {
     String jsonStr = "{\n"
                      + "  \"millisToWaitBeforeDeleting\": 1,\n"
-                     + "  \"mergeBytesLimit\": 1,\n"
-                     + "  \"mergeSegmentsLimit\" : 1,\n"
                      + "  \"maxSegmentsToMove\": 1,\n"
                      + "  \"replicantLifetime\": 1,\n"
                      + "  \"replicationThrottleLimit\": 1,\n"
@@ -408,13 +365,10 @@ public void testSerdeWithStringInKillDataSourceWhitelist() throws Exception
         1,
         1,
         1,
-        1,
-        1,
         2,
         ImmutableSet.of("test1", "test2"),
         0.1,
         Integer.MAX_VALUE,
-        false,
         1,
         ImmutableSet.of(),
         false,
@@ -427,8 +381,6 @@ public void testHandleMissingPercentOfSegmentsToConsiderPerMove() throws Excepti
   {
     String jsonStr = "{\n"
                      + "  \"millisToWaitBeforeDeleting\": 1,\n"
-                     + "  \"mergeBytesLimit\": 1,\n"
-                     + "  \"mergeSegmentsLimit\" : 1,\n"
                      + "  \"maxSegmentsToMove\": 1,\n"
                      + "  \"replicantLifetime\": 1,\n"
                      + "  \"replicationThrottleLimit\": 1,\n"
@@ -452,13 +404,10 @@ public void testHandleMissingPercentOfSegmentsToConsiderPerMove() throws Excepti
         1,
         1,
         1,
-        1,
-        1,
         2,
         whitelist,
         0.1,
         Integer.MAX_VALUE,
-        false,
         1,
         decommissioning,
         false,
@@ -471,8 +420,6 @@ public void testDeserializeWithoutMaxSegmentsInNodeLoadingQueue() throws Excepti
   {
     String jsonStr = "{\n"
                      + "  \"millisToWaitBeforeDeleting\": 1,\n"
-                     + "  \"mergeBytesLimit\": 1,\n"
-                     + "  \"mergeSegmentsLimit\" : 1,\n"
                      + "  \"maxSegmentsToMove\": 1,\n"
                      + "  \"replicantLifetime\": 1,\n"
                      + "  \"replicationThrottleLimit\": 1,\n"
@@ -492,13 +439,10 @@ public void testDeserializeWithoutMaxSegmentsInNodeLoadingQueue() throws Excepti
         1,
         1,
         1,
-        1,
-        1,
         2,
         ImmutableSet.of(),
         0.1,
         Integer.MAX_VALUE,
-        true,
         EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE,
         ImmutableSet.of(),
         false,
@@ -514,8 +458,6 @@ public void testBuilderDefaults()
     assertConfig(
         defaultConfig,
         900000,
-        524288000,
-        100,
         100,
         15,
         500,
@@ -523,7 +465,6 @@ public void testBuilderDefaults()
         emptyList,
         0.1,
         Integer.MAX_VALUE,
-        true,
         EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE,
         emptyList,
         false,
@@ -542,8 +483,6 @@ public void testBuilderWithDefaultSpecificDataSourcesToKillUnusedSegmentsInSpeci
     assertConfig(
         config,
         900000,
-        524288000,
-        100,
         100,
         15,
         500,
@@ -551,7 +490,6 @@ public void testBuilderWithDefaultSpecificDataSourcesToKillUnusedSegmentsInSpeci
         ImmutableSet.of("DATASOURCE"),
         0.1,
         Integer.MAX_VALUE,
-        false,
         EXPECTED_DEFAULT_MAX_SEGMENTS_IN_NODE_LOADING_QUEUE,
         ImmutableSet.of(),
         false,
@@ -569,26 +507,7 @@ public void testUpdate()
 
     Assert.assertEquals(
         current,
-        new CoordinatorDynamicConfig.Builder(
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null
-        ).build(current)
+        CoordinatorDynamicConfig.builder().build(current)
     );
   }
 
@@ -604,8 +523,6 @@ public void testEqualsAndHashCode()
   private void assertConfig(
       CoordinatorDynamicConfig config,
       long expectedLeadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments,
-      long expectedMergeBytesLimit,
-      int expectedMergeSegmentsLimit,
       int expectedMaxSegmentsToMove,
       int expectedReplicantLifetime,
       int expectedReplicationThrottleLimit,
@@ -613,7 +530,6 @@ private void assertConfig(
       Set<String> expectedSpecificDataSourcesToKillUnusedSegmentsIn,
       Double expectedKillTaskSlotRatio,
       @Nullable Integer expectedMaxKillTaskSlots,
-      boolean expectedKillUnusedSegmentsInAllDataSources,
       int expectedMaxSegmentsInNodeLoadingQueue,
       Set<String> decommissioningNodes,
       boolean pauseCoordination,
@@ -624,8 +540,6 @@ private void assertConfig(
         expectedLeadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments,
         config.getMarkSegmentAsUnusedDelayMillis()
     );
-    Assert.assertEquals(expectedMergeBytesLimit, config.getMergeBytesLimit());
-    Assert.assertEquals(expectedMergeSegmentsLimit, config.getMergeSegmentsLimit());
     Assert.assertEquals(expectedMaxSegmentsToMove, config.getMaxSegmentsToMove());
     Assert.assertEquals(expectedReplicantLifetime, config.getReplicantLifetime());
     Assert.assertEquals(expectedReplicationThrottleLimit, config.getReplicationThrottleLimit());
diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts
index e46d1f27c964..60ab353f8ec0 100644
--- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts
+++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.mock.ts
@@ -20,8 +20,6 @@ import type { CoordinatorDynamicConfig } from './coordinator-dynamic-config';
 
 export const DEFAULT_COORDINATOR_DYNAMIC_CONFIG: CoordinatorDynamicConfig = {
   millisToWaitBeforeDeleting: 900000,
-  mergeBytesLimit: 524288000,
-  mergeSegmentsLimit: 100,
   maxSegmentsToMove: 100,
   replicantLifetime: 15,
   replicationThrottleLimit: 500,
diff --git a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
index b60bcfc2f43c..bdd47a317225 100644
--- a/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
+++ b/web-console/src/druid-models/coordinator-dynamic-config/coordinator-dynamic-config.tsx
@@ -30,8 +30,6 @@ export interface CoordinatorDynamicConfig {
   maxKillTaskSlots?: number;
   killPendingSegmentsSkipList?: string[];
   maxSegmentsInNodeLoadingQueue?: number;
-  mergeBytesLimit?: number;
-  mergeSegmentsLimit?: number;
   millisToWaitBeforeDeleting?: number;
   replicantLifetime?: number;
   replicationThrottleLimit?: number;
@@ -220,18 +218,6 @@ export const COORDINATOR_DYNAMIC_CONFIG_FIELDS: Field<CoordinatorDynamicConfig>[
       </>
     ),
   },
-  {
-    name: 'mergeBytesLimit',
-    type: 'size-bytes',
-    defaultValue: 524288000,
-    info: <>The maximum total uncompressed size in bytes of segments to merge.</>,
-  },
-  {
-    name: 'mergeSegmentsLimit',
-    type: 'number',
-    defaultValue: 100,
-    info: <>The maximum number of segments that can be in a single append task.</>,
-  },
   {
     name: 'millisToWaitBeforeDeleting',
     type: 'number',

From 1157ecdec3705466bc595b8f56edf2a2d87e9530 Mon Sep 17 00:00:00 2001
From: Clint Wylie <cwylie@apache.org>
Date: Tue, 22 Oct 2024 19:37:17 -0700
Subject: [PATCH 06/12] abstract common base of SQL micro-benchmarks to reduce
 boilerplate and standardize parameters (#17383)

changes:
* adds `SqlBenchmarkDatasets` which contains commonly used benchmark data generator schemas
* adds `SqlBaseBenchmark` which contains common benchmark segment generation methods for any benchmark using `SqlBenchmarkDatasets`
* adds `SqlBaseQueryBenchmark` and `SqlBasePlanBenchmark` for benchmarks measuring queries and planning respectively
* migrate all existing SQL jmh benchmarks to extend `SqlBaseQueryBenchmark`, quite dramatically reducing the boilerplate needed to create benchmarks, and allowing the use of multiple datasources within a benchmark file
* adjustments to data generator stuff to allow passing in an ObjectMapper so that the same mapper can be used for both benchmark queries and segment generation, avoiding the need to register stuff with both mappers for benchmarks
* adds `SqlProjectionsBenchmark` and `SqlComplexMetricsColumnsBenchmark` for measuring projections and measuring complex metric compression respectively
---
 .../lookup/SqlReverseLookupBenchmark.java     |   8 +-
 .../benchmark/query/SqlBaseBenchmark.java     | 559 ++++++++++++++++++
 .../benchmark/query/SqlBasePlanBenchmark.java |  52 ++
 .../query/SqlBaseQueryBenchmark.java          |  50 ++
 .../druid/benchmark/query/SqlBenchmark.java   | 513 +++-------------
 .../benchmark/query/SqlBenchmarkDatasets.java | 423 +++++++++++++
 .../SqlComplexMetricsColumnsBenchmark.java    |  72 +++
 .../query/SqlExpressionBenchmark.java         | 346 ++---------
 .../benchmark/query/SqlGroupByBenchmark.java  | 310 +---------
 .../query/SqlNestedDataBenchmark.java         | 405 ++-----------
 .../benchmark/query/SqlPlanBenchmark.java     |  95 +++
 .../query/SqlProjectionsBenchmark.java        |  80 +++
 .../query/SqlWindowFunctionsBenchmark.java    | 408 ++-----------
 .../generator/GeneratorSchemaInfo.java        |  16 +
 .../org/apache/druid/segment/TestHelper.java  |  15 +
 .../segment/generator/SegmentGenerator.java   | 175 +++++-
 16 files changed, 1770 insertions(+), 1757 deletions(-)
 create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java
 create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBasePlanBenchmark.java
 create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseQueryBenchmark.java
 create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java
 create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlComplexMetricsColumnsBenchmark.java
 create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlPlanBenchmark.java
 create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlProjectionsBenchmark.java

diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/lookup/SqlReverseLookupBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/lookup/SqlReverseLookupBenchmark.java
index 9db22a2ec59d..38bcfdea0eeb 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/lookup/SqlReverseLookupBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/lookup/SqlReverseLookupBenchmark.java
@@ -20,7 +20,7 @@
 package org.apache.druid.benchmark.lookup;
 
 import com.google.common.collect.ImmutableMap;
-import org.apache.druid.benchmark.query.SqlBenchmark;
+import org.apache.druid.benchmark.query.SqlBaseBenchmark;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.Pair;
 import org.apache.druid.java.util.common.StringUtils;
@@ -54,6 +54,7 @@
 import org.openjdk.jmh.infra.Blackhole;
 
 import javax.annotation.Nullable;
+import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -111,10 +112,11 @@ public void setup()
     final QueryableIndex index =
         segmentGenerator.generate(dataSegment, schemaInfo, IndexSpec.DEFAULT, Granularities.NONE, 1);
 
-    final Pair<PlannerFactory, SqlEngine> sqlSystem = SqlBenchmark.createSqlSystem(
+    final Pair<PlannerFactory, SqlEngine> sqlSystem = SqlBaseBenchmark.createSqlSystem(
         ImmutableMap.of(dataSegment, index),
+        Collections.emptyMap(),
         ImmutableMap.of("benchmark-lookup", lookup),
-        null,
+        SqlBaseBenchmark.BenchmarkStorage.MMAP,
         closer
     );
 
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java
new file mode 100644
index 000000000000..430a6167c33c
--- /dev/null
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseBenchmark.java
@@ -0,0 +1,559 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.multibindings.MapBinder;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.frame.FrameType;
+import org.apache.druid.frame.read.FrameReader;
+import org.apache.druid.frame.segment.FrameSegment;
+import org.apache.druid.frame.testutil.FrameTestUtil;
+import org.apache.druid.guice.ExpressionModule;
+import org.apache.druid.guice.LazySingleton;
+import org.apache.druid.guice.SegmentWranglerModule;
+import org.apache.druid.guice.StartupInjectorBuilder;
+import org.apache.druid.guice.annotations.Json;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.Pair;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.java.util.common.io.Closer;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.math.expr.ExprMacroTable;
+import org.apache.druid.math.expr.ExpressionProcessing;
+import org.apache.druid.query.QueryContexts;
+import org.apache.druid.query.QueryRunnerFactoryConglomerate;
+import org.apache.druid.query.aggregation.datasketches.hll.HllSketchModule;
+import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctSqlAggregator;
+import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctUtf8SqlAggregator;
+import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchEstimateOperatorConversion;
+import org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchModule;
+import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchApproxQuantileSqlAggregator;
+import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchObjectSqlAggregator;
+import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchQuantileOperatorConversion;
+import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchQuantilesOperatorConversion;
+import org.apache.druid.query.aggregation.datasketches.theta.SketchModule;
+import org.apache.druid.query.aggregation.datasketches.theta.sql.ThetaSketchApproxCountDistinctSqlAggregator;
+import org.apache.druid.query.aggregation.datasketches.theta.sql.ThetaSketchEstimateOperatorConversion;
+import org.apache.druid.query.aggregation.datasketches.tuple.ArrayOfDoublesSketchModule;
+import org.apache.druid.query.lookup.LookupExtractor;
+import org.apache.druid.segment.IncrementalIndexSegment;
+import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.PhysicalSegmentInspector;
+import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.QueryableIndexCursorFactory;
+import org.apache.druid.segment.QueryableIndexPhysicalSegmentInspector;
+import org.apache.druid.segment.QueryableIndexSegment;
+import org.apache.druid.segment.column.StringEncodingStrategy;
+import org.apache.druid.segment.data.CompressionStrategy;
+import org.apache.druid.segment.data.FrontCodedIndexed;
+import org.apache.druid.segment.generator.SegmentGenerator;
+import org.apache.druid.segment.incremental.IncrementalIndex;
+import org.apache.druid.segment.join.JoinableFactoryWrapper;
+import org.apache.druid.server.QueryStackTests;
+import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
+import org.apache.druid.server.security.AuthConfig;
+import org.apache.druid.server.security.AuthTestUtils;
+import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest;
+import org.apache.druid.sql.calcite.aggregation.ApproxCountDistinctSqlAggregator;
+import org.apache.druid.sql.calcite.aggregation.SqlAggregationModule;
+import org.apache.druid.sql.calcite.aggregation.SqlAggregator;
+import org.apache.druid.sql.calcite.aggregation.builtin.CountSqlAggregator;
+import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
+import org.apache.druid.sql.calcite.expression.builtin.QueryLookupOperatorConversion;
+import org.apache.druid.sql.calcite.planner.CalciteRulesManager;
+import org.apache.druid.sql.calcite.planner.CatalogResolver;
+import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
+import org.apache.druid.sql.calcite.planner.DruidPlanner;
+import org.apache.druid.sql.calcite.planner.PlannerConfig;
+import org.apache.druid.sql.calcite.planner.PlannerFactory;
+import org.apache.druid.sql.calcite.planner.PlannerResult;
+import org.apache.druid.sql.calcite.run.SqlEngine;
+import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
+import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.calcite.util.LookylooModule;
+import org.apache.druid.sql.calcite.util.QueryFrameworkUtils;
+import org.apache.druid.sql.calcite.util.testoperator.CalciteTestOperatorModule;
+import org.apache.druid.sql.hook.DruidHookDispatcher;
+import org.apache.druid.timeline.DataSegment;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+
+@State(Scope.Benchmark)
+public class SqlBaseBenchmark
+{
+  private static final Logger log = new Logger(SqlBaseBenchmark.class);
+  private static final String STORAGE_MMAP = "MMAP";
+  private static final String STORAGE_INCREMENTAL = "INCREMENTAL";
+  private static final String STORAGE_FRAME_ROW = "FRAME_ROW";
+  private static final String STORAGE_FRAME_COLUMNAR = "FRAME_COLUMNAR";
+
+  static {
+    NullHandling.initializeForTests();
+    ExpressionProcessing.initializeForTests();
+    HllSketchModule.registerSerde();
+    SketchModule.registerSerde();
+    DoublesSketchModule.registerSerde();
+
+    CalciteTests.getJsonMapper()
+                .registerModules(new HllSketchModule().getJacksonModules())
+                .registerModules(new SketchModule().getJacksonModules())
+                .registerModules(new DoublesSketchModule().getJacksonModules())
+                .registerModules(new ArrayOfDoublesSketchModule().getJacksonModules());
+  }
+
+  public enum BenchmarkStorage
+  {
+    MMAP,
+    INCREMENTAL,
+    FRAME_COLUMNAR,
+    FRAME_ROW
+  }
+
+  public enum BenchmarkStringEncodingStrategy
+  {
+    UTF8,
+    FRONT_CODED_DEFAULT_V1,
+    FRONT_CODED_16_V1
+  }
+
+
+  @Param({"1500000"})
+  protected int rowsPerSegment;
+
+  @Param({
+      "false",
+      "force"
+  })
+  protected String vectorize;
+
+  @Param({
+      "UTF8",
+      "FRONT_CODED_DEFAULT_V1",
+      "FRONT_CODED_16_V1"
+  })
+  protected BenchmarkStringEncodingStrategy stringEncoding;
+
+  @Param({
+      "none",
+      "lz4"
+  })
+  protected String complexCompression;
+
+  @Param({
+      "explicit",
+      "auto"
+  })
+  protected String schemaType;
+
+  // Can be STORAGE_MMAP, STORAGE_INCREMENTAL, STORAGE_FRAME_ROW, or STORAGE_FRAME_COLUMNAR
+  @Param({
+      STORAGE_MMAP,
+      STORAGE_INCREMENTAL,
+      STORAGE_FRAME_ROW,
+      STORAGE_FRAME_COLUMNAR
+  })
+  protected BenchmarkStorage storageType;
+
+  protected SqlEngine engine;
+  @Nullable
+  protected PlannerFactory plannerFactory;
+  private final Closer closer = Closer.create();
+
+  protected QueryContexts.Vectorize vectorizeContext;
+
+
+  public String getQuery()
+  {
+    throw new UnsupportedOperationException("getQuery not implemented");
+  }
+
+  public List<String> getDatasources()
+  {
+    throw new UnsupportedOperationException("getDatasources not implemented");
+  }
+
+  protected Map<String, Object> getContext()
+  {
+    final Map<String, Object> context = ImmutableMap.of(
+        QueryContexts.VECTORIZE_KEY, vectorize,
+        QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
+    );
+    return context;
+  }
+
+  protected IndexSpec getIndexSpec()
+  {
+    return IndexSpec.builder()
+                    .withStringDictionaryEncoding(getStringEncodingStrategy())
+                    .withComplexMetricCompression(
+                        CompressionStrategy.valueOf(StringUtils.toUpperCase(complexCompression))
+                    )
+                    .build();
+  }
+
+  @Setup(Level.Trial)
+  public void setup() throws JsonProcessingException
+  {
+    vectorizeContext = QueryContexts.Vectorize.fromString(vectorize);
+    checkIncompatibleParameters();
+
+    Map<DataSegment, IncrementalIndex> realtimeSegments = new HashMap<>();
+    Map<DataSegment, QueryableIndex> segments = new HashMap<>();
+    for (String dataSource : getDatasources()) {
+      final SqlBenchmarkDatasets.BenchmarkSchema schema;
+      if ("auto".equals(schemaType)) {
+        schema = SqlBenchmarkDatasets.getSchema(dataSource).asAutoDimensions();
+      } else {
+        schema = SqlBenchmarkDatasets.getSchema(dataSource);
+      }
+
+      for (DataSegment dataSegment : schema.getDataSegments()) {
+        final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
+        log.info(
+            "Starting benchmark setup using cacheDir[%s], rows[%,d].",
+            segmentGenerator.getCacheDir(),
+            rowsPerSegment
+        );
+
+        if (BenchmarkStorage.INCREMENTAL == storageType) {
+          final IncrementalIndex index = segmentGenerator.generateIncrementalIndex(
+              dataSegment,
+              schema.getGeneratorSchemaInfo(),
+              schema.getDimensionsSpec(),
+              schema.getTransformSpec(),
+              schema.getAggregators(),
+              getIndexSpec(),
+              schema.getQueryGranularity(),
+              schema.getProjections(),
+              rowsPerSegment,
+              CalciteTests.getJsonMapper()
+          );
+          log.info(
+              "Segment metadata: %s",
+              CalciteTests.getJsonMapper().writerWithDefaultPrettyPrinter().writeValueAsString(index.getMetadata())
+          );
+          realtimeSegments.put(dataSegment, index);
+        } else {
+          final QueryableIndex index = segmentGenerator.generate(
+              dataSegment,
+              schema.getGeneratorSchemaInfo(),
+              schema.getDimensionsSpec(),
+              schema.getTransformSpec(),
+              getIndexSpec(),
+              schema.getQueryGranularity(),
+              schema.getProjections(),
+              rowsPerSegment,
+              CalciteTests.getJsonMapper()
+          );
+          log.info(
+              "Segment metadata: %s",
+              CalciteTests.getJsonMapper().writerWithDefaultPrettyPrinter().writeValueAsString(index.getMetadata())
+          );
+          segments.put(dataSegment, index);
+        }
+      }
+    }
+
+    final Pair<PlannerFactory, SqlEngine> sqlSystem = createSqlSystem(
+        segments,
+        realtimeSegments,
+        Collections.emptyMap(),
+        storageType,
+        closer
+    );
+
+    plannerFactory = sqlSystem.lhs;
+    engine = sqlSystem.rhs;
+    final ObjectMapper jsonMapper = CalciteTests.getJsonMapper();
+    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(
+        engine,
+        "EXPLAIN PLAN FOR " + getQuery(),
+        ImmutableMap.<String, Object>builder()
+                    .putAll(getContext())
+                    .put(
+                        "useNativeQueryExplain",
+                        true
+                    )
+                    .build()
+    )) {
+      final PlannerResult plannerResult = planner.plan();
+      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
+      final Object[] planResult = resultSequence.toList().get(0);
+      log.info("Native query plan:\n" +
+               jsonMapper.writerWithDefaultPrettyPrinter()
+                         .writeValueAsString(jsonMapper.readValue((String) planResult[0], List.class))
+      );
+    }
+    catch (JsonProcessingException ex) {
+      log.warn(ex, "explain failed");
+    }
+
+    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, getQuery(), getContext())) {
+      final PlannerResult plannerResult = planner.plan();
+      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
+      final int rowCount = resultSequence.toList().size();
+      log.info("Total result row count:" + rowCount);
+    }
+    catch (Throwable ex) {
+      log.warn(ex, "failed to count rows");
+    }
+
+
+    if (vectorizeContext.shouldVectorize(true)) {
+      try {
+        SqlVectorizedExpressionSanityTest.sanityTestVectorizedSqlQueries(
+            engine,
+            plannerFactory,
+            getQuery()
+        );
+        log.info("non-vectorized and vectorized results match");
+      }
+      catch (Throwable ex) {
+        log.warn(ex, "non-vectorized and vectorized results do not match");
+      }
+    }
+  }
+
+  private void checkIncompatibleParameters()
+  {
+    // if running as fork 0, maybe don't use these combinations since it will kill everything
+    if (stringEncoding != BenchmarkStringEncodingStrategy.UTF8 && storageType != BenchmarkStorage.MMAP) {
+      System.exit(0);
+    }
+    // complex compression only applies to mmap segments, dont bother otherwise
+    if (!"none".equals(complexCompression) && storageType != BenchmarkStorage.MMAP) {
+      System.exit(0);
+    }
+    // vectorize only works for mmap and frame column segments, bail out if
+    if (vectorizeContext.shouldVectorize(true) && !(storageType == BenchmarkStorage.MMAP || storageType == BenchmarkStorage.FRAME_COLUMNAR)) {
+      System.exit(0);
+    }
+  }
+
+  private StringEncodingStrategy getStringEncodingStrategy()
+  {
+    if (stringEncoding == BenchmarkStringEncodingStrategy.FRONT_CODED_DEFAULT_V1) {
+      return new StringEncodingStrategy.FrontCoded(null, FrontCodedIndexed.V1);
+    } else if (stringEncoding == BenchmarkStringEncodingStrategy.FRONT_CODED_16_V1) {
+      return new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V1);
+    } else {
+      return new StringEncodingStrategy.Utf8();
+    }
+  }
+
+  public static Pair<PlannerFactory, SqlEngine> createSqlSystem(
+      final Map<DataSegment, QueryableIndex> segmentMap,
+      final Map<DataSegment, IncrementalIndex> realtimeSegmentsMap,
+      final Map<String, LookupExtractor> lookupMap,
+      @Nullable final BenchmarkStorage storageType,
+      final Closer closer
+  )
+  {
+    final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(closer);
+    final SpecificSegmentsQuerySegmentWalker walker = SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate);
+    final PlannerConfig plannerConfig = new PlannerConfig();
+
+    for (final Map.Entry<DataSegment, QueryableIndex> segmentEntry : segmentMap.entrySet()) {
+      addSegmentToWalker(walker, segmentEntry.getKey(), segmentEntry.getValue(), storageType);
+    }
+
+    for (final Map.Entry<DataSegment, IncrementalIndex> segmentEntry : realtimeSegmentsMap.entrySet()) {
+      walker.add(
+          segmentEntry.getKey(),
+          new IncrementalIndexSegment(segmentEntry.getValue(), segmentEntry.getKey().getId())
+      );
+    }
+
+    // Child injector that adds additional lookups.
+    final Injector injector = new StartupInjectorBuilder()
+        .withEmptyProperties()
+        .add(
+            new ExpressionModule(),
+            new SegmentWranglerModule(),
+            new LookylooModule(),
+            new SqlAggregationModule(),
+            new CalciteTestOperatorModule(),
+            binder -> {
+              for (Map.Entry<String, LookupExtractor> entry : lookupMap.entrySet()) {
+                MapBinder.newMapBinder(binder, String.class, LookupExtractor.class)
+                         .addBinding(entry.getKey())
+                         .toProvider(entry::getValue)
+                         .in(LazySingleton.class);
+              }
+            },
+            new HllSketchModule(),
+            new SketchModule(),
+            new DoublesSketchModule(),
+            binder -> {
+
+            }
+        )
+        .build();
+    ObjectMapper injected = injector.getInstance(Key.get(ObjectMapper.class, Json.class));
+    injected.registerModules(new HllSketchModule().getJacksonModules());
+
+    final DruidSchemaCatalog rootSchema =
+        QueryFrameworkUtils.createMockRootSchema(
+            injector,
+            conglomerate,
+            walker,
+            plannerConfig,
+            AuthTestUtils.TEST_AUTHORIZER_MAPPER
+        );
+
+    final SqlEngine engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
+
+    final PlannerFactory plannerFactory = new PlannerFactory(
+        rootSchema,
+        createOperatorTable(injector),
+        injector.getInstance(ExprMacroTable.class),
+        plannerConfig,
+        AuthTestUtils.TEST_AUTHORIZER_MAPPER,
+        injector.getInstance(Key.get(ObjectMapper.class, Json.class)),
+        CalciteTests.DRUID_SCHEMA_NAME,
+        new CalciteRulesManager(ImmutableSet.of()),
+        new JoinableFactoryWrapper(QueryFrameworkUtils.createDefaultJoinableFactory(injector)),
+        CatalogResolver.NULL_RESOLVER,
+        new AuthConfig(),
+        new DruidHookDispatcher()
+    );
+
+    return Pair.of(plannerFactory, engine);
+  }
+
+  private static void addSegmentToWalker(
+      final SpecificSegmentsQuerySegmentWalker walker,
+      final DataSegment descriptor,
+      final QueryableIndex index,
+      @Nullable final BenchmarkStorage storageType
+  )
+  {
+    if (storageType == null || BenchmarkStorage.MMAP == storageType) {
+      walker.add(descriptor, new QueryableIndexSegment(index, descriptor.getId()));
+    } else if (BenchmarkStorage.FRAME_ROW == storageType) {
+      QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+      walker.add(
+          descriptor,
+          new FrameSegment(
+              FrameTestUtil.cursorFactoryToFrame(cursorFactory, FrameType.ROW_BASED),
+              FrameReader.create(cursorFactory.getRowSignature()),
+              descriptor.getId()
+          )
+          {
+            @Nullable
+            @Override
+            public <T> T as(@Nonnull Class<T> clazz)
+            {
+              // computed sql schema uses segment metadata, which relies on physical inspector, use the underlying index
+              if (clazz.equals(PhysicalSegmentInspector.class)) {
+                return (T) new QueryableIndexPhysicalSegmentInspector(index);
+              }
+              return super.as(clazz);
+            }
+          }
+      );
+    } else if (BenchmarkStorage.FRAME_COLUMNAR == storageType) {
+      QueryableIndexCursorFactory cursorFactory = new QueryableIndexCursorFactory(index);
+      walker.add(
+          descriptor,
+          new FrameSegment(
+              FrameTestUtil.cursorFactoryToFrame(cursorFactory, FrameType.COLUMNAR),
+              FrameReader.create(cursorFactory.getRowSignature()),
+              descriptor.getId()
+          )
+          {
+            @Nullable
+            @Override
+            public <T> T as(@Nonnull Class<T> clazz)
+            {
+              // computed sql schema uses segment metadata, which relies on physical inspector, use the underlying index
+              if (clazz.equals(PhysicalSegmentInspector.class)) {
+                return (T) new QueryableIndexPhysicalSegmentInspector(index);
+              }
+              return super.as(clazz);
+            }
+          }
+      );
+    } else {
+      throw new IAE("Invalid storageType[%s]", storageType);
+    }
+  }
+
+  private static DruidOperatorTable createOperatorTable(final Injector injector)
+  {
+    try {
+      final Set<SqlOperatorConversion> operators = new HashSet<>();
+      operators.add(injector.getInstance(QueryLookupOperatorConversion.class));
+      operators.addAll(
+          ImmutableList.of(
+              new HllSketchEstimateOperatorConversion(),
+              new ThetaSketchEstimateOperatorConversion(),
+              new DoublesSketchQuantileOperatorConversion(),
+              new DoublesSketchQuantilesOperatorConversion()
+          )
+      );
+      final ApproxCountDistinctSqlAggregator countDistinctSqlAggregator =
+          new ApproxCountDistinctSqlAggregator(new HllSketchApproxCountDistinctSqlAggregator());
+      final Set<SqlAggregator> aggregators = new HashSet<>(
+          ImmutableList.of(
+              new DoublesSketchApproxQuantileSqlAggregator(),
+              new DoublesSketchObjectSqlAggregator(),
+              new HllSketchApproxCountDistinctSqlAggregator(),
+              new HllSketchApproxCountDistinctUtf8SqlAggregator(),
+              new ThetaSketchApproxCountDistinctSqlAggregator(),
+              new CountSqlAggregator(countDistinctSqlAggregator),
+              countDistinctSqlAggregator
+          )
+      );
+      return new DruidOperatorTable(aggregators, operators);
+    }
+    catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @TearDown(Level.Trial)
+  public void tearDown() throws Exception
+  {
+    closer.close();
+  }
+}
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBasePlanBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBasePlanBenchmark.java
new file mode 100644
index 000000000000..e3ef0c67399b
--- /dev/null
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBasePlanBenchmark.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.query.QueryContexts;
+import org.apache.druid.sql.calcite.planner.DruidPlanner;
+import org.apache.druid.sql.calcite.planner.PlannerResult;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public class SqlBasePlanBenchmark extends SqlBaseBenchmark
+{
+  @Benchmark
+  @BenchmarkMode(Mode.AverageTime)
+  @OutputTimeUnit(TimeUnit.MILLISECONDS)
+  public void planSql(Blackhole blackhole)
+  {
+    final Map<String, Object> context = ImmutableMap.of(
+        QueryContexts.VECTORIZE_KEY, vectorize,
+        QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
+    );
+    final String sql = getQuery();
+    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, context)) {
+      final PlannerResult plannerResult = planner.plan();
+      blackhole.consume(plannerResult);
+    }
+  }
+}
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseQueryBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseQueryBenchmark.java
new file mode 100644
index 000000000000..48a8b49b5bce
--- /dev/null
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBaseQueryBenchmark.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import org.apache.druid.java.util.common.guava.Sequence;
+import org.apache.druid.sql.calcite.planner.DruidPlanner;
+import org.apache.druid.sql.calcite.planner.PlannerResult;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public class SqlBaseQueryBenchmark extends SqlBaseBenchmark
+{
+  @Benchmark
+  @BenchmarkMode(Mode.AverageTime)
+  @OutputTimeUnit(TimeUnit.MILLISECONDS)
+  public void querySql(Blackhole blackhole)
+  {
+    final Map<String, Object> context = getContext();
+    final String sql = getQuery();
+    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, context)) {
+      final PlannerResult plannerResult = planner.plan();
+      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
+      final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
+      blackhole.consume(lastRow);
+    }
+  }
+}
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
index 0e38757f12b1..09a54698de02 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java
@@ -19,101 +19,15 @@
 
 package org.apache.druid.benchmark.query;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import com.google.inject.multibindings.MapBinder;
-import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.frame.FrameType;
-import org.apache.druid.frame.testutil.FrameTestUtil;
-import org.apache.druid.guice.ExpressionModule;
-import org.apache.druid.guice.LazySingleton;
-import org.apache.druid.guice.SegmentWranglerModule;
-import org.apache.druid.guice.StartupInjectorBuilder;
-import org.apache.druid.guice.annotations.Json;
-import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Yielder;
-import org.apache.druid.java.util.common.guava.Yielders;
-import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.math.expr.ExprMacroTable;
-import org.apache.druid.query.QueryContexts;
-import org.apache.druid.query.QueryRunnerFactoryConglomerate;
-import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctSqlAggregator;
-import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctUtf8SqlAggregator;
-import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchApproxQuantileSqlAggregator;
-import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchObjectSqlAggregator;
-import org.apache.druid.query.aggregation.datasketches.theta.sql.ThetaSketchApproxCountDistinctSqlAggregator;
-import org.apache.druid.query.lookup.LookupExtractor;
-import org.apache.druid.segment.AutoTypeColumnSchema;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.QueryableIndexCursorFactory;
-import org.apache.druid.segment.QueryableIndexSegment;
-import org.apache.druid.segment.column.StringEncodingStrategy;
-import org.apache.druid.segment.data.FrontCodedIndexed;
-import org.apache.druid.segment.generator.GeneratorBasicSchemas;
-import org.apache.druid.segment.generator.GeneratorSchemaInfo;
-import org.apache.druid.segment.generator.SegmentGenerator;
-import org.apache.druid.segment.join.JoinableFactoryWrapper;
-import org.apache.druid.segment.transform.TransformSpec;
-import org.apache.druid.server.QueryStackTests;
-import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
-import org.apache.druid.server.security.AuthConfig;
-import org.apache.druid.server.security.AuthTestUtils;
-import org.apache.druid.sql.calcite.aggregation.ApproxCountDistinctSqlAggregator;
-import org.apache.druid.sql.calcite.aggregation.SqlAggregationModule;
-import org.apache.druid.sql.calcite.aggregation.SqlAggregator;
-import org.apache.druid.sql.calcite.aggregation.builtin.CountSqlAggregator;
-import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
-import org.apache.druid.sql.calcite.expression.builtin.QueryLookupOperatorConversion;
-import org.apache.druid.sql.calcite.planner.CalciteRulesManager;
-import org.apache.druid.sql.calcite.planner.CatalogResolver;
-import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
-import org.apache.druid.sql.calcite.planner.DruidPlanner;
-import org.apache.druid.sql.calcite.planner.PlannerConfig;
-import org.apache.druid.sql.calcite.planner.PlannerFactory;
-import org.apache.druid.sql.calcite.planner.PlannerResult;
-import org.apache.druid.sql.calcite.run.SqlEngine;
-import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
-import org.apache.druid.sql.calcite.util.CalciteTests;
-import org.apache.druid.sql.calcite.util.LookylooModule;
-import org.apache.druid.sql.calcite.util.QueryFrameworkUtils;
-import org.apache.druid.sql.calcite.util.testoperator.CalciteTestOperatorModule;
-import org.apache.druid.sql.hook.DruidHookDispatcher;
-import org.apache.druid.timeline.DataSegment;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
 
-import javax.annotation.Nullable;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /**
  * Benchmark that tests various SQL queries.
@@ -122,40 +36,30 @@
 @Fork(value = 1)
 @Warmup(iterations = 3)
 @Measurement(iterations = 5)
-public class SqlBenchmark
+public class SqlBenchmark extends SqlBaseQueryBenchmark
 {
-  static {
-    NullHandling.initializeForTests();
-  }
-
-  private static final Logger log = new Logger(SqlBenchmark.class);
-
-  private static final String STORAGE_MMAP = "mmap";
-  private static final String STORAGE_FRAME_ROW = "frame-row";
-  private static final String STORAGE_FRAME_COLUMNAR = "frame-columnar";
-
-  private static final List<String> QUERIES = ImmutableList.of(
+  static final List<String> QUERIES = ImmutableList.of(
       // 0, 1, 2, 3: Timeseries, unfiltered
-      "SELECT COUNT(*) FROM foo",
-      "SELECT COUNT(DISTINCT hyper) FROM foo",
-      "SELECT SUM(sumLongSequential), SUM(sumFloatNormal) FROM foo",
-      "SELECT FLOOR(__time TO MINUTE), SUM(sumLongSequential), SUM(sumFloatNormal) FROM foo GROUP BY 1",
+      "SELECT COUNT(*) FROM druid.basic",
+      "SELECT APPROX_COUNT_DISTINCT_BUILTIN(hyper) FROM druid.basic",
+      "SELECT SUM(sumLongSequential), SUM(sumFloatNormal) FROM druid.basic",
+      "SELECT FLOOR(__time TO MINUTE), SUM(sumLongSequential), SUM(sumFloatNormal) FROM druid.basic GROUP BY 1",
 
       // 4: Timeseries, low selectivity filter (90% of rows match)
-      "SELECT SUM(sumLongSequential), SUM(sumFloatNormal) FROM foo WHERE dimSequential NOT LIKE '%3'",
+      "SELECT SUM(sumLongSequential), SUM(sumFloatNormal) FROM druid.basic WHERE dimSequential NOT LIKE '%3'",
 
       // 5: Timeseries, high selectivity filter (0.1% of rows match)
-      "SELECT SUM(sumLongSequential), SUM(sumFloatNormal) FROM foo WHERE dimSequential = '311'",
+      "SELECT SUM(sumLongSequential), SUM(sumFloatNormal) FROM druid.basic WHERE dimSequential = '311'",
 
       // 6: Timeseries, mixing low selectivity index-capable filter (90% of rows match) + cursor filter
-      "SELECT SUM(sumLongSequential), SUM(sumFloatNormal) FROM foo\n"
+      "SELECT SUM(sumLongSequential), SUM(sumFloatNormal) FROM druid.basic\n"
       + "WHERE dimSequential NOT LIKE '%3' AND maxLongUniform > 10",
 
       // 7: Timeseries, low selectivity toplevel filter (90%), high selectivity filtered aggregator (0.1%)
       "SELECT\n"
       + "  SUM(sumLongSequential) FILTER(WHERE dimSequential = '311'),\n"
       + "  SUM(sumFloatNormal)\n"
-      + "FROM foo\n"
+      + "FROM druid.basic\n"
       + "WHERE dimSequential NOT LIKE '%3'",
 
       // 8: Timeseries, no toplevel filter, various filtered aggregators with clauses repeated.
@@ -175,7 +79,7 @@ public class SqlBenchmark
       + "  COUNT(*) FILTER(WHERE dimSequential LIKE '%3'),\n"
       + "  COUNT(*) FILTER(WHERE dimSequential NOT LIKE '%3'),\n"
       + "  COUNT(*)\n"
-      + "FROM foo",
+      + "FROM druid.basic",
 
       // 9: Timeseries, toplevel time filter, time-comparison filtered aggregators
       "SELECT\n"
@@ -183,233 +87,233 @@ public class SqlBenchmark
       + "    FILTER(WHERE __time >= TIMESTAMP '2000-01-01 00:00:00' AND __time < TIMESTAMP '2000-01-01 12:00:00'),\n"
       + "  SUM(sumLongSequential)\n"
       + "    FILTER(WHERE __time >= TIMESTAMP '2000-01-01 12:00:00' AND __time < TIMESTAMP '2000-01-02 00:00:00')\n"
-      + "FROM foo\n"
+      + "FROM druid.basic\n"
       + "WHERE __time >= TIMESTAMP '2000-01-01 00:00:00' AND __time < TIMESTAMP '2000-01-02 00:00:00'",
 
       // 10, 11: GroupBy two strings, unfiltered, unordered
-      "SELECT dimSequential, dimZipf, SUM(sumLongSequential) FROM foo GROUP BY 1, 2",
-      "SELECT dimSequential, dimZipf, SUM(sumLongSequential), COUNT(*) FROM foo GROUP BY 1, 2",
+      "SELECT dimSequential, dimZipf, SUM(sumLongSequential) FROM druid.basic GROUP BY 1, 2",
+      "SELECT dimSequential, dimZipf, SUM(sumLongSequential), COUNT(*) FROM druid.basic GROUP BY 1, 2",
 
       // 12, 13, 14: GroupBy one string, unfiltered, various aggregator configurations
-      "SELECT dimZipf FROM foo GROUP BY 1",
-      "SELECT dimZipf, COUNT(*) FROM foo GROUP BY 1 ORDER BY COUNT(*) DESC",
-      "SELECT dimZipf, SUM(sumLongSequential), COUNT(*) FROM foo GROUP BY 1 ORDER BY COUNT(*) DESC",
+      "SELECT dimZipf FROM druid.basic GROUP BY 1",
+      "SELECT dimZipf, COUNT(*) FROM druid.basic GROUP BY 1 ORDER BY COUNT(*) DESC",
+      "SELECT dimZipf, SUM(sumLongSequential), COUNT(*) FROM druid.basic GROUP BY 1 ORDER BY COUNT(*) DESC",
 
       // 15, 16: GroupBy long, unfiltered, unordered; with and without aggregators
-      "SELECT maxLongUniform FROM foo GROUP BY 1",
-      "SELECT maxLongUniform, SUM(sumLongSequential), COUNT(*) FROM foo GROUP BY 1",
+      "SELECT maxLongUniform FROM druid.basic GROUP BY 1",
+      "SELECT maxLongUniform, SUM(sumLongSequential), COUNT(*) FROM druid.basic GROUP BY 1",
 
       // 17, 18: GroupBy long, filter by long, unordered; with and without aggregators
-      "SELECT maxLongUniform FROM foo WHERE maxLongUniform > 10 GROUP BY 1",
-      "SELECT maxLongUniform, SUM(sumLongSequential), COUNT(*) FROM foo WHERE maxLongUniform > 10 GROUP BY 1",
+      "SELECT maxLongUniform FROM druid.basic WHERE maxLongUniform > 10 GROUP BY 1",
+      "SELECT maxLongUniform, SUM(sumLongSequential), COUNT(*) FROM druid.basic WHERE maxLongUniform > 10 GROUP BY 1",
       // 19: ultra mega union matrix
       "WITH matrix (dimZipf, dimSequential) AS (\n"
       + "  (\n"
       + "    SELECT '100', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '100'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '110', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '110'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '120', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '120'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '130', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '130'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '140', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '140'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '150', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '150'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '160', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '160'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '170', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '170'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '180', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '180'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '190', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '190'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '200', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '200'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '210', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '210'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '220', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '220'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '230', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '230'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '240', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '240'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '250', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '250'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '260', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '260'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '270', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '270'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '280', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '280'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '290', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '290'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '300', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '300'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '310', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '310'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '320', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '320'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '330', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '330'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '340', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '340'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '350', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '350'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '360', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '360'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '370', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '370'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT '380', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE dimZipf = '380'\n"
       + "    GROUP BY dimSequential\n"
       + "  )\n"
       + "UNION ALL\n"
       + "  (\n"
       + "    SELECT 'other', dimSequential\n"
-      + "    FROM (SELECT * FROM foo WHERE dimUniform != 1)\n"
+      + "    FROM (SELECT * FROM druid.basic WHERE dimUniform != 1)\n"
       + "    WHERE\n"
       + "      dimZipf NOT IN (\n"
       + "        '100', '110', '120', '130', '140', '150', '160', '170', '180', '190',\n"
@@ -423,65 +327,45 @@ public class SqlBenchmark
 
       // 20: GroupBy, doubles sketches
       "SELECT dimZipf, APPROX_QUANTILE_DS(sumFloatNormal, 0.5), DS_QUANTILES_SKETCH(maxLongUniform) "
-      + "FROM foo "
+      + "FROM druid.basic "
       + "GROUP BY 1",
 
       // 21, 22: stringy stuff
-      "SELECT dimSequential, dimZipf, SUM(sumLongSequential) FROM foo WHERE dimUniform NOT LIKE '%3' GROUP BY 1, 2",
-      "SELECT dimZipf, SUM(sumLongSequential) FROM foo WHERE dimSequential = '311' GROUP BY 1 ORDER BY 1",
+      "SELECT dimSequential, dimZipf, SUM(sumLongSequential) FROM druid.basic WHERE dimUniform NOT LIKE '%3' GROUP BY 1, 2",
+      "SELECT dimZipf, SUM(sumLongSequential) FROM druid.basic WHERE dimSequential = '311' GROUP BY 1 ORDER BY 1",
       // 23: full scan
-      "SELECT * FROM foo",
-      "SELECT * FROM foo WHERE dimSequential IN ('1', '2', '3', '4', '5', '10', '11', '20', '21', '23', '40', '50', '64', '70', '100')",
-      "SELECT * FROM foo WHERE dimSequential > '10' AND dimSequential < '8500'",
-      "SELECT dimSequential, dimZipf, SUM(sumLongSequential) FROM foo WHERE dimSequential IN ('1', '2', '3', '4', '5', '10', '11', '20', '21', '23', '40', '50', '64', '70', '100') GROUP BY 1, 2",
-      "SELECT dimSequential, dimZipf, SUM(sumLongSequential) FROM foo WHERE dimSequential > '10' AND dimSequential < '8500' GROUP BY 1, 2",
+      "SELECT * FROM druid.basic",
+      "SELECT * FROM druid.basic WHERE dimSequential IN ('1', '2', '3', '4', '5', '10', '11', '20', '21', '23', '40', '50', '64', '70', '100')",
+      "SELECT * FROM druid.basic WHERE dimSequential > '10' AND dimSequential < '8500'",
+      "SELECT dimSequential, dimZipf, SUM(sumLongSequential) FROM druid.basic WHERE dimSequential IN ('1', '2', '3', '4', '5', '10', '11', '20', '21', '23', '40', '50', '64', '70', '100') GROUP BY 1, 2",
+      "SELECT dimSequential, dimZipf, SUM(sumLongSequential) FROM druid.basic WHERE dimSequential > '10' AND dimSequential < '8500' GROUP BY 1, 2",
 
       // 28, 29, 30, 31: Approximate count distinct of strings
-      "SELECT APPROX_COUNT_DISTINCT_BUILTIN(dimZipf) FROM foo",
-      "SELECT APPROX_COUNT_DISTINCT_DS_HLL(dimZipf) FROM foo",
-      "SELECT APPROX_COUNT_DISTINCT_DS_HLL_UTF8(dimZipf) FROM foo",
-      "SELECT APPROX_COUNT_DISTINCT_DS_THETA(dimZipf) FROM foo",
+      "SELECT APPROX_COUNT_DISTINCT_BUILTIN(dimZipf) FROM druid.basic",
+      "SELECT APPROX_COUNT_DISTINCT_DS_HLL(dimZipf) FROM druid.basic",
+      "SELECT APPROX_COUNT_DISTINCT_DS_HLL_UTF8(dimZipf) FROM druid.basic",
+      "SELECT APPROX_COUNT_DISTINCT_DS_THETA(dimZipf) FROM druid.basic",
       // 32: LATEST aggregator long
-      "SELECT LATEST(long1) FROM foo",
+      "SELECT LATEST(long1) FROM druid.expressions",
       // 33: LATEST aggregator double
-      "SELECT LATEST(double4) FROM foo",
+      "SELECT LATEST(double4) FROM druid.expressions",
       // 34: LATEST aggregator double
-      "SELECT LATEST(float3) FROM foo",
+      "SELECT LATEST(float3) FROM druid.expressions",
       // 35: LATEST aggregator double
-      "SELECT LATEST(float3), LATEST(long1), LATEST(double4) FROM foo",
+      "SELECT LATEST(float3), LATEST(long1), LATEST(double4) FROM druid.expressions",
       // 36,37: filter numeric nulls
-      "SELECT SUM(long5) FROM foo WHERE long5 IS NOT NULL",
-      "SELECT string2, SUM(long5) FROM foo WHERE long5 IS NOT NULL GROUP BY 1",
+      "SELECT SUM(long5) FROM druid.expressions WHERE long5 IS NOT NULL",
+      "SELECT string2, SUM(long5) FROM druid.expressions WHERE long5 IS NOT NULL GROUP BY 1",
       // 38: EARLIEST aggregator long
-      "SELECT EARLIEST(long1) FROM foo",
+      "SELECT EARLIEST(long1) FROM druid.expressions",
       // 39: EARLIEST aggregator double
-      "SELECT EARLIEST(double4) FROM foo",
+      "SELECT EARLIEST(double4) FROM druid.expressions",
       // 40: EARLIEST aggregator float
-      "SELECT EARLIEST(float3) FROM foo",
+      "SELECT EARLIEST(float3) FROM druid.expressions",
       // 41: nested OR filter
-      "SELECT dimSequential, COUNT(*) from foo WHERE dimSequential = '1' AND (dimMultivalEnumerated IN ('Hello', 'World', 'Foo', 'Bar', 'Baz') OR sumLongSequential = 1) GROUP BY 1"
+      "SELECT dimSequential, COUNT(*) from druid.basic WHERE dimSequential = '1' AND (dimMultivalEnumerated IN ('Hello', 'World', 'druid.basic', 'Bar', 'Baz') OR sumLongSequential = 1) GROUP BY 1"
   );
 
-  @Param({"5000000"})
-  private int rowsPerSegment;
-
-  // Can be "false", "true", or "force"
-  @Param({"false", "force"})
-  private String vectorize;
-
-  // Can be "none" or "front-coded-N"
-  @Param({
-      "none",
-      "front-coded-4"
-  })
-  private String stringEncoding;
-
-  @Param({
-      "explicit",
-      "auto"
-  })
-  private String schema;
-
   @Param({
       "0",
       "1",
@@ -528,254 +412,15 @@ public class SqlBenchmark
   })
   private String query;
 
-  // Can be STORAGE_MMAP, STORAGE_FRAME_ROW, or STORAGE_FRAME_COLUMNAR
-  @Param({STORAGE_MMAP})
-  private String storageType;
-
-  private SqlEngine engine;
-
-  @Nullable
-  private PlannerFactory plannerFactory;
-  private final Closer closer = Closer.create();
-
-  @Setup(Level.Trial)
-  public void setup()
-  {
-    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("basic");
-    final DataSegment dataSegment = schemaInfo.makeSegmentDescriptor("foo");
-    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
-
-    log.info("Starting benchmark setup using cacheDir[%s], rows[%,d].", segmentGenerator.getCacheDir(), rowsPerSegment);
-    final QueryableIndex index;
-    if ("auto".equals(schema)) {
-      List<DimensionSchema> columnSchemas = schemaInfo.getDimensionsSpec()
-                                                      .getDimensions()
-                                                      .stream()
-                                                      .map(x -> new AutoTypeColumnSchema(x.getName(), null))
-                                                      .collect(Collectors.toList());
-      index = segmentGenerator.generate(
-          dataSegment,
-          schemaInfo,
-          DimensionsSpec.builder().setDimensions(columnSchemas).build(),
-          TransformSpec.NONE,
-          IndexSpec.builder().withStringDictionaryEncoding(getStringEncodingStrategy()).build(),
-          Granularities.NONE,
-          rowsPerSegment
-      );
-    } else {
-      index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment);
-    }
-
-    final Pair<PlannerFactory, SqlEngine> sqlSystem = createSqlSystem(
-        ImmutableMap.of(dataSegment, index),
-        Collections.emptyMap(),
-        null,
-        closer
-    );
-
-    plannerFactory = sqlSystem.lhs;
-    engine = sqlSystem.rhs;
-
-    final String sql = QUERIES.get(Integer.parseInt(query));
-    final ObjectMapper jsonMapper = CalciteTests.getJsonMapper();
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, "EXPLAIN PLAN FOR " + sql, ImmutableMap.of("useNativeQueryExplain", true))) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Object[] planResult = resultSequence.toList().get(0);
-      log.info("Native query plan:\n" +
-               jsonMapper.writerWithDefaultPrettyPrinter()
-                         .writeValueAsString(jsonMapper.readValue((String) planResult[0], List.class))
-      );
-    }
-    catch (JsonProcessingException ignored) {
-
-    }
-
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, ImmutableMap.of())) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Yielder<Object[]> yielder = Yielders.each(resultSequence);
-      int rowCounter = 0;
-      while (!yielder.isDone()) {
-        rowCounter++;
-        yielder.next(yielder.get());
-      }
-      log.info("Total result row count:" + rowCounter);
-    }
-    catch (Throwable ignored) {
-
-    }
-  }
-
-  private StringEncodingStrategy getStringEncodingStrategy()
-  {
-    if (stringEncoding.startsWith("front-coded")) {
-      String[] split = stringEncoding.split("-");
-      int bucketSize = Integer.parseInt(split[2]);
-      return new StringEncodingStrategy.FrontCoded(bucketSize, FrontCodedIndexed.V1);
-    } else {
-      return new StringEncodingStrategy.Utf8();
-    }
-  }
-
-  public static Pair<PlannerFactory, SqlEngine> createSqlSystem(
-      final Map<DataSegment, QueryableIndex> segmentMap,
-      final Map<String, LookupExtractor> lookupMap,
-      @Nullable final String storageType,
-      final Closer closer
-  )
-  {
-    final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(closer);
-    final SpecificSegmentsQuerySegmentWalker walker = SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate);
-    final PlannerConfig plannerConfig = new PlannerConfig();
-
-    for (final Map.Entry<DataSegment, QueryableIndex> segmentEntry : segmentMap.entrySet()) {
-      addSegmentToWalker(walker, segmentEntry.getKey(), segmentEntry.getValue(), storageType);
-    }
-
-    // Child injector that adds additional lookups.
-    final Injector injector = new StartupInjectorBuilder()
-        .withEmptyProperties()
-        .add(
-            new ExpressionModule(),
-            new SegmentWranglerModule(),
-            new LookylooModule(),
-            new SqlAggregationModule(),
-            new CalciteTestOperatorModule(),
-            binder -> {
-              for (Map.Entry<String, LookupExtractor> entry : lookupMap.entrySet()) {
-                MapBinder.newMapBinder(binder, String.class, LookupExtractor.class)
-                         .addBinding(entry.getKey())
-                         .toProvider(entry::getValue)
-                         .in(LazySingleton.class);
-              }
-            }
-        )
-        .build();
-
-    final DruidSchemaCatalog rootSchema =
-        QueryFrameworkUtils.createMockRootSchema(
-            injector,
-            conglomerate,
-            walker,
-            plannerConfig,
-            AuthTestUtils.TEST_AUTHORIZER_MAPPER
-        );
-
-    final SqlEngine engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
-
-    final PlannerFactory plannerFactory = new PlannerFactory(
-        rootSchema,
-        createOperatorTable(injector),
-        injector.getInstance(ExprMacroTable.class),
-        plannerConfig,
-        AuthTestUtils.TEST_AUTHORIZER_MAPPER,
-        injector.getInstance(Key.get(ObjectMapper.class, Json.class)),
-        CalciteTests.DRUID_SCHEMA_NAME,
-        new CalciteRulesManager(ImmutableSet.of()),
-        new JoinableFactoryWrapper(QueryFrameworkUtils.createDefaultJoinableFactory(injector)),
-        CatalogResolver.NULL_RESOLVER,
-        new AuthConfig(),
-        new DruidHookDispatcher()
-    );
-
-    return Pair.of(plannerFactory, engine);
-  }
-
-  private static void addSegmentToWalker(
-      final SpecificSegmentsQuerySegmentWalker walker,
-      final DataSegment descriptor,
-      final QueryableIndex index,
-      @Nullable final String storageType
-  )
-  {
-    if (storageType == null || STORAGE_MMAP.equals(storageType)) {
-      walker.add(descriptor, new QueryableIndexSegment(index, descriptor.getId()));
-    } else if (STORAGE_FRAME_ROW.equals(storageType)) {
-      walker.add(
-          descriptor,
-          FrameTestUtil.cursorFactoryToFrameSegment(
-              new QueryableIndexCursorFactory(index),
-              FrameType.ROW_BASED,
-              descriptor.getId()
-          )
-      );
-    } else if (STORAGE_FRAME_COLUMNAR.equals(storageType)) {
-      walker.add(
-          descriptor,
-          FrameTestUtil.cursorFactoryToFrameSegment(
-              new QueryableIndexCursorFactory(index),
-              FrameType.COLUMNAR,
-              descriptor.getId()
-          )
-      );
-    } else {
-      throw new IAE("Invalid storageType[%s]", storageType);
-    }
-  }
-
-  private static DruidOperatorTable createOperatorTable(final Injector injector)
-  {
-    try {
-      final Set<SqlOperatorConversion> extractionOperators = new HashSet<>();
-      extractionOperators.add(injector.getInstance(QueryLookupOperatorConversion.class));
-      final ApproxCountDistinctSqlAggregator countDistinctSqlAggregator =
-          new ApproxCountDistinctSqlAggregator(new HllSketchApproxCountDistinctSqlAggregator());
-      final Set<SqlAggregator> aggregators = new HashSet<>(
-          ImmutableList.of(
-              new DoublesSketchApproxQuantileSqlAggregator(),
-              new DoublesSketchObjectSqlAggregator(),
-              new HllSketchApproxCountDistinctSqlAggregator(),
-              new HllSketchApproxCountDistinctUtf8SqlAggregator(),
-              new ThetaSketchApproxCountDistinctSqlAggregator(),
-              new CountSqlAggregator(countDistinctSqlAggregator),
-              countDistinctSqlAggregator
-          )
-      );
-      return new DruidOperatorTable(aggregators, extractionOperators);
-    }
-    catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
-  @TearDown(Level.Trial)
-  public void tearDown() throws Exception
-  {
-    closer.close();
-  }
-
-  @Benchmark
-  @BenchmarkMode(Mode.AverageTime)
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  public void querySql(Blackhole blackhole)
+  @Override
+  public String getQuery()
   {
-    final Map<String, Object> context = ImmutableMap.of(
-        QueryContexts.VECTORIZE_KEY, vectorize,
-        QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
-    );
-    final String sql = QUERIES.get(Integer.parseInt(query));
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, context)) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
-      blackhole.consume(lastRow);
-    }
+    return QUERIES.get(Integer.parseInt(query));
   }
 
-  @Benchmark
-  @BenchmarkMode(Mode.AverageTime)
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  public void planSql(Blackhole blackhole)
+  @Override
+  public List<String> getDatasources()
   {
-    final Map<String, Object> context = ImmutableMap.of(
-        QueryContexts.VECTORIZE_KEY, vectorize,
-        QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
-    );
-    final String sql = QUERIES.get(Integer.parseInt(query));
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, context)) {
-      final PlannerResult plannerResult = planner.plan();
-      blackhole.consume(plannerResult);
-    }
+    return ImmutableList.of(SqlBenchmarkDatasets.BASIC);
   }
 }
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java
new file mode 100644
index 000000000000..7b6b48802f34
--- /dev/null
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmarkDatasets.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import org.apache.druid.data.input.impl.AggregateProjectionSpec;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.LongDimensionSchema;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.granularity.Granularity;
+import org.apache.druid.query.aggregation.AggregatorFactory;
+import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
+import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
+import org.apache.druid.query.aggregation.datasketches.hll.HllSketchBuildAggregatorFactory;
+import org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchAggregatorFactory;
+import org.apache.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory;
+import org.apache.druid.query.expression.TestExprMacroTable;
+import org.apache.druid.segment.AutoTypeColumnSchema;
+import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.generator.GeneratorBasicSchemas;
+import org.apache.druid.segment.generator.GeneratorSchemaInfo;
+import org.apache.druid.segment.transform.ExpressionTransform;
+import org.apache.druid.segment.transform.TransformSpec;
+import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.partition.LinearShardSpec;
+import org.joda.time.Interval;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class SqlBenchmarkDatasets
+{
+  private static Map<String, BenchmarkSchema> DATASET_SCHEMAS = new HashMap<>();
+
+  public static String BASIC = "basic";
+  public static String EXPRESSIONS = "expressions";
+  public static String NESTED = "nested";
+  public static String DATASKETCHES = "datasketches";
+  public static String PROJECTIONS = "projections";
+  public static String GROUPER = "grouper";
+
+  // initialize all benchmark dataset schemas to feed the data generators when running benchmarks, add any additional
+  // datasets to this initializer as needed and they will be available to any benchmarks at the table name added here
+  static {
+    // the classic 'basic' schema, string dimension oriented with a few metrics
+    final GeneratorSchemaInfo basicSchema = GeneratorBasicSchemas.SCHEMA_MAP.get(GeneratorBasicSchemas.BASIC_SCHEMA);
+    DATASET_SCHEMAS.put(
+        BASIC,
+        new BenchmarkSchema(
+            Collections.singletonList(makeSegment(BASIC, basicSchema.getDataInterval())),
+            basicSchema,
+            TransformSpec.NONE,
+            makeDimensionsSpec(basicSchema),
+            basicSchema.getAggsArray(),
+            Collections.emptyList(),
+            Granularities.NONE
+        )
+    );
+
+    // expression testbench schema, lots of different column types
+    final GeneratorSchemaInfo expressionsSchema = GeneratorBasicSchemas.SCHEMA_MAP.get(
+        GeneratorBasicSchemas.EXPRESSION_TESTBENCH_SCHEMA
+    );
+    DATASET_SCHEMAS.put(
+        EXPRESSIONS,
+        new BenchmarkSchema(
+            Collections.singletonList(makeSegment(EXPRESSIONS, expressionsSchema.getDataInterval())),
+            expressionsSchema,
+            TransformSpec.NONE,
+            makeDimensionsSpec(expressionsSchema),
+            expressionsSchema.getAggsArray(),
+            Collections.emptyList(),
+            Granularities.NONE
+        )
+    );
+
+    // expressions schema but with transform to create nested column
+    DATASET_SCHEMAS.put(
+        NESTED,
+        new BenchmarkSchema(
+            Collections.singletonList(makeSegment(NESTED, expressionsSchema.getDataInterval())),
+            expressionsSchema,
+            new TransformSpec(
+                null,
+                ImmutableList.of(
+                    new ExpressionTransform(
+                        "nested",
+                        "json_object('long1', long1, 'nesteder', json_object('string1', string1, 'long2', long2, 'double3',double3, 'string5', string5))",
+                        TestExprMacroTable.INSTANCE
+                    )
+                )
+            ),
+            DimensionsSpec.builder().setDimensions(
+                ImmutableList.copyOf(
+                    Iterables.concat(
+                        expressionsSchema.getDimensionsSpecExcludeAggs().getDimensions(),
+                        Collections.singletonList(new AutoTypeColumnSchema("nested", null))
+                    )
+                )
+            ).build(),
+            expressionsSchema.getAggsArray(),
+            Collections.emptyList(),
+            Granularities.NONE
+        )
+    );
+
+    // expressions schema but with some datasketch aggs defined
+    GeneratorSchemaInfo datasketchesSchema = new GeneratorSchemaInfo(
+            expressionsSchema.getColumnSchemas(),
+            ImmutableList.of(
+                new HllSketchBuildAggregatorFactory("hll_string5", "string5", null, null, null, false, true),
+                new SketchMergeAggregatorFactory("theta_string5", "string5", null, null, null, null),
+                new DoublesSketchAggregatorFactory("quantiles_float4", "float4", null, null, null),
+                new DoublesSketchAggregatorFactory("quantiles_long3", "long3", null, null, null)
+            ),
+            expressionsSchema.getDataInterval(),
+            true
+    );
+    DATASET_SCHEMAS.put(
+        DATASKETCHES,
+        new BenchmarkSchema(
+            Collections.singletonList(makeSegment(DATASKETCHES, datasketchesSchema.getDataInterval())),
+            datasketchesSchema,
+            TransformSpec.NONE,
+            makeDimensionsSpec(datasketchesSchema),
+            datasketchesSchema.getAggsArray(),
+            Collections.emptyList(),
+            Granularities.NONE
+        )
+    );
+
+    // expressions schema with projections
+    DATASET_SCHEMAS.put(
+        PROJECTIONS,
+        new BenchmarkSchema(
+            Collections.singletonList(makeSegment(PROJECTIONS, expressionsSchema.getDataInterval())),
+            expressionsSchema,
+            TransformSpec.NONE,
+            makeDimensionsSpec(expressionsSchema),
+            expressionsSchema.getAggsArray(),
+            Arrays.asList(
+                new AggregateProjectionSpec(
+                    "string2_hourly_sums_hll",
+                    VirtualColumns.create(
+                        Granularities.toVirtualColumn(Granularities.HOUR, "__gran")
+                    ),
+                    Arrays.asList(
+                        new StringDimensionSchema("string2"),
+                        new LongDimensionSchema("__gran")
+                    ),
+                    new AggregatorFactory[]{
+                        new LongSumAggregatorFactory("long4_sum", "long4"),
+                        new DoubleSumAggregatorFactory("double2_sum", "double2"),
+                        new HllSketchBuildAggregatorFactory("hll_string5", "string5", null, null, null, false, true)
+                    }
+                ),
+                new AggregateProjectionSpec(
+                    "string2_long2_sums",
+                    VirtualColumns.EMPTY,
+                    Arrays.asList(
+                        new StringDimensionSchema("string2"),
+                        new LongDimensionSchema("long2")
+                    ),
+                    new AggregatorFactory[]{
+                        new LongSumAggregatorFactory("long4_sum", "long4"),
+                        new DoubleSumAggregatorFactory("double2_sum", "double2"),
+                        new HllSketchBuildAggregatorFactory("hll_string5", "string5", null, null, null, false, true)
+                    }
+                )
+            ),
+            Granularities.NONE
+        )
+    );
+
+    // group-by testing, 2 segments
+    final GeneratorSchemaInfo groupingSchema = GeneratorBasicSchemas.SCHEMA_MAP.get(
+        GeneratorBasicSchemas.GROUPBY_TESTBENCH_SCHEMA
+    );
+    DATASET_SCHEMAS.put(
+        GROUPER,
+        new BenchmarkSchema(
+            Arrays.asList(
+                makeSegment(GROUPER, groupingSchema.getDataInterval(), 0),
+                makeSegment(GROUPER, groupingSchema.getDataInterval(), 1)
+            ),
+            groupingSchema,
+            new TransformSpec(
+                null,
+                ImmutableList.of(
+                    // string array dims
+                    new ExpressionTransform(
+                        "stringArray-Sequential-100_000",
+                        "array(\"string-Sequential-100_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    new ExpressionTransform(
+                        "stringArray-Sequential-3_000_000",
+                        "array(\"string-Sequential-10_000_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    /*
+                    new ExpressionTransform(
+                        "stringArray-Sequential-1_000_000_000",
+                        "array(\"string-Sequential-1_000_000_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),*/
+                    new ExpressionTransform(
+                        "stringArray-ZipF-1_000_000",
+                        "array(\"string-ZipF-1_000_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    new ExpressionTransform(
+                        "stringArray-Uniform-1_000_000",
+                        "array(\"string-Uniform-1_000_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),
+
+                    // long array dims
+                    new ExpressionTransform(
+                        "longArray-Sequential-100_000",
+                        "array(\"long-Sequential-100_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    new ExpressionTransform(
+                        "longArray-Sequential-3_000_000",
+                        "array(\"long-Sequential-10_000_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    /*
+                    new ExpressionTransform(
+                        "longArray-Sequential-1_000_000_000",
+                        "array(\"long-Sequential-1_000_000_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),*/
+                    new ExpressionTransform(
+                        "longArray-ZipF-1_000_000",
+                        "array(\"long-ZipF-1_000_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    new ExpressionTransform(
+                        "longArray-Uniform-1_000_000",
+                        "array(\"long-Uniform-1_000_000\")",
+                        TestExprMacroTable.INSTANCE
+                    ),
+
+                    // nested complex json dim
+                    new ExpressionTransform(
+                        "nested-Sequential-100_000",
+                        "json_object('long1', \"long-Sequential-100_000\", 'nesteder', json_object('long1', \"long-Sequential-100_000\"))",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    new ExpressionTransform(
+                        "nested-Sequential-3_000_000",
+                        "json_object('long1', \"long-Sequential-10_000_000\", 'nesteder', json_object('long1', \"long-Sequential-10_000_000\"))",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    /*
+                    new ExpressionTransform(
+                        "nested-Sequential-1_000_000_000",
+                        "json_object('long1', \"long-Sequential-1_000_000_000\", 'nesteder', json_object('long1', \"long-Sequential-1_000_000_000\"))",
+                        TestExprMacroTable.INSTANCE
+                    ),*/
+                    new ExpressionTransform(
+                        "nested-ZipF-1_000_000",
+                        "json_object('long1', \"long-ZipF-1_000_000\", 'nesteder', json_object('long1', \"long-ZipF-1_000_000\"))",
+                        TestExprMacroTable.INSTANCE
+                    ),
+                    new ExpressionTransform(
+                        "nested-Uniform-1_000_000",
+                        "json_object('long1', \"long-Uniform-1_000_000\", 'nesteder', json_object('long1', \"long-Uniform-1_000_000\"))",
+                        TestExprMacroTable.INSTANCE
+                    )
+                )
+            ),
+            makeDimensionsSpec(groupingSchema),
+            groupingSchema.getAggsArray(),
+            Collections.emptyList(),
+            Granularities.NONE
+        )
+    );
+  }
+
+  public static BenchmarkSchema getSchema(String dataset)
+  {
+    return DATASET_SCHEMAS.get(dataset);
+  }
+
+  private static DataSegment makeSegment(String datasource, Interval interval)
+  {
+    return makeSegment(datasource, interval, 0);
+  }
+
+  private static DataSegment makeSegment(String datasource, Interval interval, int partitionNumber)
+  {
+    return DataSegment.builder()
+                      .dataSource(datasource)
+                      .interval(interval)
+                      .version("1")
+                      .shardSpec(new LinearShardSpec(partitionNumber))
+                      .size(0)
+                      .build();
+  }
+
+  private static DimensionsSpec makeDimensionsSpec(GeneratorSchemaInfo schemaInfo)
+  {
+    return DimensionsSpec.builder().setDimensions(schemaInfo.getDimensionsSpecExcludeAggs().getDimensions()).build();
+  }
+
+  public static class BenchmarkSchema
+  {
+    private final List<DataSegment> dataSegments;
+    private final GeneratorSchemaInfo generatorSchemaInfo;
+    private final TransformSpec transformSpec;
+    private final DimensionsSpec dimensionsSpec;
+    private final AggregatorFactory[] aggregators;
+    private final Granularity queryGranularity;
+    private final List<AggregateProjectionSpec> projections;
+
+    public BenchmarkSchema(
+        List<DataSegment> dataSegments,
+        GeneratorSchemaInfo generatorSchemaInfo,
+        TransformSpec transformSpec,
+        DimensionsSpec dimensionSpec,
+        AggregatorFactory[] aggregators,
+        List<AggregateProjectionSpec> projections,
+        Granularity queryGranularity
+    )
+    {
+      this.dataSegments = dataSegments;
+      this.generatorSchemaInfo = generatorSchemaInfo;
+      this.transformSpec = transformSpec;
+      this.dimensionsSpec = dimensionSpec;
+      this.aggregators = aggregators;
+      this.queryGranularity = queryGranularity;
+      this.projections = projections;
+    }
+
+    public List<DataSegment> getDataSegments()
+    {
+      return dataSegments;
+    }
+
+    public GeneratorSchemaInfo getGeneratorSchemaInfo()
+    {
+      return generatorSchemaInfo;
+    }
+
+    public TransformSpec getTransformSpec()
+    {
+      return transformSpec;
+    }
+
+    public DimensionsSpec getDimensionsSpec()
+    {
+      return dimensionsSpec;
+    }
+
+    public AggregatorFactory[] getAggregators()
+    {
+      return aggregators;
+    }
+
+    public Granularity getQueryGranularity()
+    {
+      return queryGranularity;
+    }
+
+    public List<AggregateProjectionSpec> getProjections()
+    {
+      return projections;
+    }
+
+    public BenchmarkSchema asAutoDimensions()
+    {
+      return new SqlBenchmarkDatasets.BenchmarkSchema(
+          dataSegments,
+          generatorSchemaInfo,
+          transformSpec,
+          dimensionsSpec.withDimensions(
+                    dimensionsSpec.getDimensions()
+                                  .stream()
+                                  .map(dim -> new AutoTypeColumnSchema(dim.getName(), null))
+                                  .collect(Collectors.toList())
+          ),
+          aggregators,
+          projections.stream()
+                     .map(projection -> new AggregateProjectionSpec(
+                         projection.getName(),
+                         projection.getVirtualColumns(),
+                         projection.getGroupingColumns()
+                                   .stream()
+                                   .map(dim -> new AutoTypeColumnSchema(dim.getName(), null))
+                                   .collect(Collectors.toList()),
+                         projection.getAggregators()
+                     )).collect(Collectors.toList()),
+          queryGranularity
+      );
+    }
+  }
+}
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlComplexMetricsColumnsBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlComplexMetricsColumnsBenchmark.java
new file mode 100644
index 000000000000..3c001c9d0397
--- /dev/null
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlComplexMetricsColumnsBenchmark.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableList;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.List;
+
+@State(Scope.Benchmark)
+@Fork(value = 1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+public class SqlComplexMetricsColumnsBenchmark extends SqlBaseQueryBenchmark
+{
+  private static final List<String> QUERIES = ImmutableList.of(
+      "SELECT APPROX_COUNT_DISTINCT_DS_HLL(hll_string5) FROM druid.datasketches",
+      "SELECT APPROX_COUNT_DISTINCT_DS_THETA(theta_string5) FROM druid.datasketches",
+      "SELECT DS_GET_QUANTILE(DS_QUANTILES_SKETCH(quantiles_float4), 0.5) FROM druid.datasketches",
+      "SELECT DS_GET_QUANTILE(DS_QUANTILES_SKETCH(quantiles_long3), 0.9) FROM druid.datasketches",
+      "SELECT string2, APPROX_COUNT_DISTINCT_DS_HLL(hll_string5) FROM druid.datasketches GROUP BY 1 ORDER BY 2 DESC",
+      "SELECT string2, APPROX_COUNT_DISTINCT_DS_THETA(theta_string5, 4096) FROM druid.datasketches GROUP BY 1 ORDER BY 2 DESC",
+      "SELECT string2, DS_GET_QUANTILE(DS_QUANTILES_SKETCH(quantiles_float4), 0.5) FROM druid.datasketches GROUP BY 1 ORDER BY 2 DESC",
+      "SELECT string2, DS_GET_QUANTILE(DS_QUANTILES_SKETCH(quantiles_long3), 0.9) FROM druid.datasketches GROUP BY 1 ORDER BY 2 DESC"
+  );
+
+  @Param({
+      "0",
+      "1",
+      "2",
+      "3",
+      "4",
+      "5",
+      "6",
+      "7"
+  })
+  private String query;
+
+  @Override
+  public String getQuery()
+  {
+    return QUERIES.get(Integer.parseInt(query));
+  }
+
+  @Override
+  public List<String> getDatasources()
+  {
+    return ImmutableList.of(SqlBenchmarkDatasets.DATASKETCHES);
+  }
+}
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
index cc6dd636df08..46036fcc64ff 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java
@@ -19,69 +19,19 @@
 
 package org.apache.druid.benchmark.query;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Yielder;
-import org.apache.druid.java.util.common.guava.Yielders;
-import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.math.expr.ExpressionProcessing;
-import org.apache.druid.query.DruidProcessingConfig;
 import org.apache.druid.query.QueryContexts;
-import org.apache.druid.query.QueryRunnerFactoryConglomerate;
 import org.apache.druid.query.groupby.GroupByQueryConfig;
-import org.apache.druid.segment.AutoTypeColumnSchema;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.generator.GeneratorBasicSchemas;
-import org.apache.druid.segment.generator.GeneratorSchemaInfo;
-import org.apache.druid.segment.generator.SegmentGenerator;
-import org.apache.druid.segment.transform.TransformSpec;
-import org.apache.druid.server.QueryStackTests;
-import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
-import org.apache.druid.server.security.AuthConfig;
-import org.apache.druid.server.security.AuthTestUtils;
-import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest;
-import org.apache.druid.sql.calcite.planner.CalciteRulesManager;
-import org.apache.druid.sql.calcite.planner.CatalogResolver;
-import org.apache.druid.sql.calcite.planner.DruidPlanner;
-import org.apache.druid.sql.calcite.planner.PlannerConfig;
-import org.apache.druid.sql.calcite.planner.PlannerFactory;
-import org.apache.druid.sql.calcite.planner.PlannerResult;
-import org.apache.druid.sql.calcite.run.SqlEngine;
-import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
-import org.apache.druid.sql.calcite.util.CalciteTests;
-import org.apache.druid.sql.hook.DruidHookDispatcher;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.partition.LinearShardSpec;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
 
-import javax.annotation.Nullable;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /**
  * Benchmark that tests various SQL queries.
@@ -90,157 +40,107 @@
 @Fork(value = 1)
 @Warmup(iterations = 3)
 @Measurement(iterations = 5)
-public class SqlExpressionBenchmark
+public class SqlExpressionBenchmark extends SqlBaseQueryBenchmark
 {
-  private static final Logger log = new Logger(SqlExpressionBenchmark.class);
-
-  static {
-    NullHandling.initializeForTests();
-    ExpressionProcessing.initializeForTests();
-  }
-
-  private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig()
-  {
-    @Override
-    public int intermediateComputeSizeBytes()
-    {
-      return 512 * 1024 * 1024;
-    }
-
-    @Override
-    public int getNumMergeBuffers()
-    {
-      return 3;
-    }
-
-    @Override
-    public int getNumThreads()
-    {
-      return 1;
-    }
-
-    @Override
-    public String getFormatString()
-    {
-      return "benchmarks-processing-%s";
-    }
-  };
-
-
   private static final List<String> QUERIES = ImmutableList.of(
       // ===========================
       // non-expression reference queries
       // ===========================
       // 0: non-expression timeseries reference, 1 columns
-      "SELECT SUM(long1) FROM foo",
+      "SELECT SUM(long1) FROM expressions",
       // 1: non-expression timeseries reference, 2 columns
-      "SELECT SUM(long1), SUM(long2) FROM foo",
+      "SELECT SUM(long1), SUM(long2) FROM expressions",
       // 2: non-expression timeseries reference, 3 columns
-      "SELECT SUM(long1), SUM(long4), SUM(double1) FROM foo",
+      "SELECT SUM(long1), SUM(long4), SUM(double1) FROM expressions",
       // 3: non-expression timeseries reference, 4 columns
-      "SELECT SUM(long1), SUM(long4), SUM(double1), SUM(float3) FROM foo",
+      "SELECT SUM(long1), SUM(long4), SUM(double1), SUM(float3) FROM expressions",
       // 4: non-expression timeseries reference, 5 columns
-      "SELECT SUM(long1), SUM(long4), SUM(double1), SUM(float3), SUM(long5) FROM foo",
+      "SELECT SUM(long1), SUM(long4), SUM(double1), SUM(float3), SUM(long5) FROM expressions",
       // 5: group by non-expr with 1 agg
-      "SELECT string2, SUM(long1) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT string2, SUM(long1) FROM expressions GROUP BY 1 ORDER BY 2",
       // 6: group by non-expr with 2 agg
-      "SELECT string2, SUM(long1), SUM(double3) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT string2, SUM(long1), SUM(double3) FROM expressions GROUP BY 1 ORDER BY 2",
       // ===========================
       // expressions
       // ===========================
       // 7: math op - 2 longs
-      "SELECT SUM(long1 * long2) FROM foo",
+      "SELECT SUM(long1 * long2) FROM expressions",
       // 8: mixed math - 2 longs, 1 double
-      "SELECT SUM((long1 * long2) / double1) FROM foo",
+      "SELECT SUM((long1 * long2) / double1) FROM expressions",
       // 9: mixed math - 2 longs, 1 double, 1 float
-      "SELECT SUM(float3 + ((long1 * long4)/double1)) FROM foo",
+      "SELECT SUM(float3 + ((long1 * long4)/double1)) FROM expressions",
       // 10: mixed math - 3 longs, 1 double, 1 float
-      "SELECT SUM(long5 - (float3 + ((long1 * long4)/double1))) FROM foo",
+      "SELECT SUM(long5 - (float3 + ((long1 * long4)/double1))) FROM expressions",
       // 11: all same math op - 3 longs, 1 double, 1 float
-      "SELECT SUM(long5 * float3 * long1 * long4 * double1) FROM foo",
+      "SELECT SUM(long5 * float3 * long1 * long4 * double1) FROM expressions",
       // 12: cos
-      "SELECT cos(double2) FROM foo",
+      "SELECT cos(double2) FROM expressions",
       // 13: unary negate
-      "SELECT SUM(-long4) FROM foo",
+      "SELECT SUM(-long4) FROM expressions",
       // 14: string long
-      "SELECT SUM(PARSE_LONG(string1)) FROM foo",
+      "SELECT SUM(PARSE_LONG(string1)) FROM expressions",
       // 15: string longer
-      "SELECT SUM(PARSE_LONG(string3)) FROM foo",
+      "SELECT SUM(PARSE_LONG(string3)) FROM expressions",
       // 16: time floor, non-expr col + reg agg
-      "SELECT TIME_FLOOR(__time, 'PT1H'), string2, SUM(double4) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT TIME_FLOOR(__time, 'PT1H'), string2, SUM(double4) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 17: time floor, non-expr col + expr agg
-      "SELECT TIME_FLOOR(__time, 'PT1H'), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT TIME_FLOOR(__time, 'PT1H'), string2, SUM(long1 * double4) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 18: time floor + non-expr agg (timeseries) (non-expression reference)
-      "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1) FROM foo GROUP BY 1 ORDER BY 1",
+      "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1) FROM expressions GROUP BY 1 ORDER BY 1",
       // 19: time floor + expr agg (timeseries)
-      "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 1",
+      "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1 * long4) FROM expressions GROUP BY 1 ORDER BY 1",
       // 20: time floor + non-expr agg (group by)
-      "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1) FROM expressions GROUP BY 1 ORDER BY 2",
       // 21: time floor + expr agg (group by)
-      "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT TIME_FLOOR(__time, 'PT1H'), SUM(long1 * long4) FROM expressions GROUP BY 1 ORDER BY 2",
       // 22: time floor offset by 1 day + non-expr agg (group by)
-      "SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'PT1H'), SUM(long1) FROM foo GROUP BY 1 ORDER BY 1",
+      "SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'PT1H'), SUM(long1) FROM expressions GROUP BY 1 ORDER BY 1",
       // 23: time floor offset by 1 day + expr agg (group by)
-      "SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'PT1H'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 1",
+      "SELECT TIME_FLOOR(TIMESTAMPADD(DAY, -1, __time), 'PT1H'), SUM(long1 * long4) FROM expressions GROUP BY 1 ORDER BY 1",
       // 24: group by long expr with non-expr agg
-      "SELECT (long1 * long2), SUM(double1) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT (long1 * long2), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
       // 25: group by non-expr with expr agg
-      "SELECT string2, SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT string2, SUM(long1 * long4) FROM expressions GROUP BY 1 ORDER BY 2",
       // 26: group by string expr with non-expr agg
-      "SELECT CONCAT(string2, '-', long2), SUM(double1) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT CONCAT(string2, '-', long2), SUM(double1) FROM expressions GROUP BY 1 ORDER BY 2",
       // 27: group by string expr with expr agg
-      "SELECT CONCAT(string2, '-', long2), SUM(long1 * double4) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT CONCAT(string2, '-', long2), SUM(long1 * double4) FROM expressions GROUP BY 1 ORDER BY 2",
       // 28: group by single input string low cardinality expr with expr agg
-      "SELECT CONCAT(string2, '-', 'foo'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT CONCAT(string2, '-', 'expressions'), SUM(long1 * long4) FROM expressions GROUP BY 1 ORDER BY 2",
       // 29: group by single input string high cardinality expr with expr agg
-      "SELECT CONCAT(string3, '-', 'foo'), SUM(long1 * long4) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT CONCAT(string3, '-', 'expressions'), SUM(long1 * long4) FROM expressions GROUP BY 1 ORDER BY 2",
       // 30: logical and operator
-      "SELECT CAST(long1 as BOOLEAN) AND CAST (long2 as BOOLEAN), COUNT(*) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT CAST(long1 as BOOLEAN) AND CAST (long2 as BOOLEAN), COUNT(*) FROM expressions GROUP BY 1 ORDER BY 2",
       // 31: isnull, notnull
-      "SELECT long5 IS NULL, long3 IS NOT NULL, count(*) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT long5 IS NULL, long3 IS NOT NULL, count(*) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 32: time shift, non-expr col + reg agg, regular
-      "SELECT TIME_SHIFT(__time, 'PT1H', 3), string2, SUM(double4) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT TIME_SHIFT(__time, 'PT1H', 3), string2, SUM(double4) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 33: time shift, non-expr col + expr agg, sequential low cardinality
-      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long1), 'PT1H', 1), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long1), 'PT1H', 1), string2, SUM(long1 * double4) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 34: time shift + non-expr agg (timeseries) (non-expression reference), zipf distribution low cardinality
-      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long2), 'PT1H', 1), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long2), 'PT1H', 1), string2, SUM(long1 * double4) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 35: time shift + expr agg (timeseries), zipf distribution high cardinality
-      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long3), 'PT1H', 1), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long3), 'PT1H', 1), string2, SUM(long1 * double4) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 36: time shift + non-expr agg (group by), uniform distribution low cardinality
-      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long4), 'PT1H', 1), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long4), 'PT1H', 1), string2, SUM(long1 * double4) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 37: time shift + expr agg (group by), uniform distribution high cardinality
-      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long5), 'PT1H', 1), string2, SUM(long1 * double4) FROM foo GROUP BY 1,2 ORDER BY 3",
+      "SELECT TIME_SHIFT(MILLIS_TO_TIMESTAMP(long5), 'PT1H', 1), string2, SUM(long1 * double4) FROM expressions GROUP BY 1,2 ORDER BY 3",
       // 38,39: array element filtering
-      "SELECT string1, long1 FROM foo WHERE ARRAY_CONTAINS(\"multi-string3\", 100) GROUP BY 1,2",
-      "SELECT string1, long1 FROM foo WHERE ARRAY_OVERLAP(\"multi-string3\", ARRAY[100, 200]) GROUP BY 1,2",
+      "SELECT string1, long1 FROM expressions WHERE ARRAY_CONTAINS(\"multi-string3\", 100) GROUP BY 1,2",
+      "SELECT string1, long1 FROM expressions WHERE ARRAY_OVERLAP(\"multi-string3\", ARRAY[100, 200]) GROUP BY 1,2",
       // 40: regex filtering
-      "SELECT string4, COUNT(*) FROM foo WHERE REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL GROUP BY 1",
+      "SELECT string4, COUNT(*) FROM expressions WHERE REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL GROUP BY 1",
       // 41: complicated filtering
-      "SELECT string2, SUM(long1) FROM foo WHERE string1 = '1000' AND string5 LIKE '%1%' AND (string3 in ('1', '10', '20', '22', '32') AND long2 IN (1, 19, 21, 23, 25, 26, 46) AND double3 < 1010.0 AND double3 > 1000.0 AND (string4 = '1' OR REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL)) GROUP BY 1 ORDER BY 2",
+      "SELECT string2, SUM(long1) FROM expressions WHERE string1 = '1000' AND string5 LIKE '%1%' AND (string3 in ('1', '10', '20', '22', '32') AND long2 IN (1, 19, 21, 23, 25, 26, 46) AND double3 < 1010.0 AND double3 > 1000.0 AND (string4 = '1' OR REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL)) GROUP BY 1 ORDER BY 2",
       // 42: array_contains expr
-      "SELECT ARRAY_CONTAINS(\"multi-string3\", 100) FROM foo",
-      "SELECT ARRAY_CONTAINS(\"multi-string3\", ARRAY[1, 2, 10, 11, 20, 22, 30, 33, 40, 44, 50, 55, 100]) FROM foo",
-      "SELECT ARRAY_OVERLAP(\"multi-string3\", ARRAY[1, 100]) FROM foo",
-      "SELECT ARRAY_OVERLAP(\"multi-string3\", ARRAY[1, 2, 10, 11, 20, 22, 30, 33, 40, 44, 50, 55, 100]) FROM foo",
+      "SELECT ARRAY_CONTAINS(\"multi-string3\", 100) FROM expressions",
+      "SELECT ARRAY_CONTAINS(\"multi-string3\", ARRAY[1, 2, 10, 11, 20, 22, 30, 33, 40, 44, 50, 55, 100]) FROM expressions",
+      "SELECT ARRAY_OVERLAP(\"multi-string3\", ARRAY[1, 100]) FROM expressions",
+      "SELECT ARRAY_OVERLAP(\"multi-string3\", ARRAY[1, 2, 10, 11, 20, 22, 30, 33, 40, 44, 50, 55, 100]) FROM expressions",
       // 46: filters with random orders
-      "SELECT string2, SUM(long1) FROM foo WHERE string5 LIKE '%1%' AND string1 = '1000' GROUP BY 1 ORDER BY 2",
-      "SELECT string2, SUM(long1) FROM foo WHERE string5 LIKE '%1%' AND (string3 in ('1', '10', '20', '22', '32') AND long2 IN (1, 19, 21, 23, 25, 26, 46) AND double3 < 1010.0 AND double3 > 1000.0 AND (string4 = '1' OR REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL)) AND string1 = '1000' GROUP BY 1 ORDER BY 2"
-      );
-
-  @Param({"5000000"})
-  private int rowsPerSegment;
-
-  @Param({
-      "false",
-      "force"
-  })
-  private String vectorize;
-
-  @Param({
-      "explicit",
-      "auto"
-  })
-  private String schema;
+      "SELECT string2, SUM(long1) FROM expressions WHERE string5 LIKE '%1%' AND string1 = '1000' GROUP BY 1 ORDER BY 2",
+      "SELECT string2, SUM(long1) FROM expressions WHERE string5 LIKE '%1%' AND (string3 in ('1', '10', '20', '22', '32') AND long2 IN (1, 19, 21, 23, 25, 26, 46) AND double3 < 1010.0 AND double3 > 1000.0 AND (string4 = '1' OR REGEXP_EXTRACT(string1, '^1') IS NOT NULL OR REGEXP_EXTRACT('Z' || string2, '^Z2') IS NOT NULL)) AND string1 = '1000' GROUP BY 1 ORDER BY 2"
+  );
 
   @Param({
       "singleString",
@@ -304,154 +204,26 @@ public String getFormatString()
   })
   private String query;
 
-  private SqlEngine engine;
-  @Nullable
-  private PlannerFactory plannerFactory;
-  private Closer closer = Closer.create();
-
-  @Setup(Level.Trial)
-  public void setup()
+  @Override
+  public String getQuery()
   {
-    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
-
-    final DataSegment dataSegment = DataSegment.builder()
-                                               .dataSource("foo")
-                                               .interval(schemaInfo.getDataInterval())
-                                               .version("1")
-                                               .shardSpec(new LinearShardSpec(0))
-                                               .size(0)
-                                               .build();
-
-    final PlannerConfig plannerConfig = new PlannerConfig();
-
-    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
-    log.info(
-        "Starting benchmark setup using cacheDir[%s], rows[%,d], schema[%s].",
-        segmentGenerator.getCacheDir(),
-        rowsPerSegment,
-        schema
-    );
-    final QueryableIndex index;
-    if ("auto".equals(schema)) {
-      List<DimensionSchema> columnSchemas = schemaInfo.getDimensionsSpec()
-                                                      .getDimensions()
-                                                      .stream()
-                                                      .map(x -> new AutoTypeColumnSchema(x.getName(), null))
-                                                      .collect(Collectors.toList());
-      index = segmentGenerator.generate(
-          dataSegment,
-          schemaInfo,
-          DimensionsSpec.builder().setDimensions(columnSchemas).build(),
-          TransformSpec.NONE,
-          IndexSpec.DEFAULT,
-          Granularities.NONE,
-          rowsPerSegment
-      );
-    } else {
-      index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment);
-    }
-
-    final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(
-        closer,
-        PROCESSING_CONFIG
-    );
-
-    final SpecificSegmentsQuerySegmentWalker walker = SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate).add(
-        dataSegment,
-        index
-    );
-    closer.register(walker);
-    final ObjectMapper jsonMapper = CalciteTests.getJsonMapper();
-    final DruidSchemaCatalog rootSchema =
-        CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
-    engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
-    plannerFactory = new PlannerFactory(
-        rootSchema,
-        CalciteTests.createOperatorTable(),
-        CalciteTests.createExprMacroTable(),
-        plannerConfig,
-        AuthTestUtils.TEST_AUTHORIZER_MAPPER,
-        jsonMapper,
-        CalciteTests.DRUID_SCHEMA_NAME,
-        new CalciteRulesManager(ImmutableSet.of()),
-        CalciteTests.createJoinableFactoryWrapper(),
-        CatalogResolver.NULL_RESOLVER,
-        new AuthConfig(),
-        new DruidHookDispatcher()
-    );
-
-    try {
-      SqlVectorizedExpressionSanityTest.sanityTestVectorizedSqlQueries(
-          engine,
-          plannerFactory,
-          QUERIES.get(Integer.parseInt(query))
-      );
-      log.info("non-vectorized and vectorized results match");
-    }
-    catch (Throwable ex) {
-      log.warn(ex, "non-vectorized and vectorized results do not match");
-    }
-
-    final String sql = QUERIES.get(Integer.parseInt(query));
-
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(
-        engine,
-        "EXPLAIN PLAN FOR " + sql,
-        ImmutableMap.of(
-            "useNativeQueryExplain",
-            true
-        )
-    )) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Object[] planResult = resultSequence.toList().get(0);
-      log.info("Native query plan:\n" +
-               jsonMapper.writerWithDefaultPrettyPrinter()
-                         .writeValueAsString(jsonMapper.readValue((String) planResult[0], List.class))
-      );
-    }
-    catch (JsonProcessingException ex) {
-      log.warn(ex, "explain failed");
-    }
-
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, ImmutableMap.of())) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Yielder<Object[]> yielder = Yielders.each(resultSequence);
-      int rowCounter = 0;
-      while (!yielder.isDone()) {
-        rowCounter++;
-        yielder.next(yielder.get());
-      }
-      log.info("Total result row count:" + rowCounter);
-    }
-    catch (Throwable ex) {
-      log.warn(ex, "failed to count rows");
-    }
+    return QUERIES.get(Integer.parseInt(query));
   }
 
-  @TearDown(Level.Trial)
-  public void tearDown() throws Exception
+  @Override
+  public List<String> getDatasources()
   {
-    closer.close();
+    return ImmutableList.of(SqlBenchmarkDatasets.EXPRESSIONS);
   }
 
-  @Benchmark
-  @BenchmarkMode(Mode.AverageTime)
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  public void querySql(Blackhole blackhole)
+  @Override
+  protected Map<String, Object> getContext()
   {
     final Map<String, Object> context = ImmutableMap.of(
         QueryContexts.VECTORIZE_KEY, vectorize,
         QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize,
         GroupByQueryConfig.CTX_KEY_DEFER_EXPRESSION_DIMENSIONS, deferExpressionDimensions
     );
-    final String sql = QUERIES.get(Integer.parseInt(query));
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, context)) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
-      blackhole.consume(lastRow);
-    }
+    return context;
   }
 }
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java
index 1f58f97b8e8a..fbb4d1c394d4 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlGroupByBenchmark.java
@@ -19,109 +19,23 @@
 
 package org.apache.druid.benchmark.query;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.guice.BuiltInTypesModule;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.math.expr.ExpressionProcessing;
-import org.apache.druid.query.DruidProcessingConfig;
-import org.apache.druid.query.QueryRunnerFactoryConglomerate;
-import org.apache.druid.query.expression.TestExprMacroTable;
-import org.apache.druid.segment.AutoTypeColumnSchema;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.column.StringEncodingStrategy;
-import org.apache.druid.segment.generator.GeneratorBasicSchemas;
-import org.apache.druid.segment.generator.GeneratorSchemaInfo;
-import org.apache.druid.segment.generator.SegmentGenerator;
-import org.apache.druid.segment.transform.ExpressionTransform;
-import org.apache.druid.segment.transform.TransformSpec;
-import org.apache.druid.server.QueryStackTests;
-import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
-import org.apache.druid.server.security.AuthConfig;
-import org.apache.druid.server.security.AuthTestUtils;
-import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest;
-import org.apache.druid.sql.calcite.planner.CalciteRulesManager;
-import org.apache.druid.sql.calcite.planner.CatalogResolver;
-import org.apache.druid.sql.calcite.planner.DruidPlanner;
-import org.apache.druid.sql.calcite.planner.PlannerConfig;
-import org.apache.druid.sql.calcite.planner.PlannerFactory;
-import org.apache.druid.sql.calcite.planner.PlannerResult;
-import org.apache.druid.sql.calcite.run.SqlEngine;
-import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
-import org.apache.druid.sql.calcite.util.CalciteTests;
-import org.apache.druid.sql.hook.DruidHookDispatcher;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.partition.LinearShardSpec;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
 
-import javax.annotation.Nullable;
 import java.util.Collections;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 @State(Scope.Benchmark)
 @Fork(value = 1)
 @Warmup(iterations = 3)
 @Measurement(iterations = 5)
-public class SqlGroupByBenchmark
+public class SqlGroupByBenchmark extends SqlBaseQueryBenchmark
 {
-  static {
-    NullHandling.initializeForTests();
-    ExpressionProcessing.initializeForTests();
-    BuiltInTypesModule.registerHandlersAndSerde();
-  }
-
-  private static final Logger log = new Logger(SqlGroupByBenchmark.class);
-
-  private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig()
-  {
-    @Override
-    public int intermediateComputeSizeBytes()
-    {
-      return 512 * 1024 * 1024;
-    }
-
-    @Override
-    public int getNumMergeBuffers()
-    {
-      return 3;
-    }
-
-    @Override
-    public int getNumThreads()
-    {
-      return 1;
-    }
-
-    @Override
-    public String getFormatString()
-    {
-      return "benchmarks-processing-%s";
-    }
-  };
-
   @Param({
       "string-Sequential-100_000",
       "string-Sequential-10_000_000",
@@ -167,226 +81,16 @@ public String getFormatString()
   })
   private String groupingDimension;
 
-  private SqlEngine engine;
-  @Nullable
-  private PlannerFactory plannerFactory;
-  private Closer closer = Closer.create();
-
-  @Setup(Level.Trial)
-  public void setup()
-  {
-    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("groupBy-testbench");
-
-    final DataSegment dataSegment = DataSegment.builder()
-                                               .dataSource("foo")
-                                               .interval(schemaInfo.getDataInterval())
-                                               .version("1")
-                                               .shardSpec(new LinearShardSpec(0))
-                                               .size(0)
-                                               .build();
-    final DataSegment dataSegment2 = DataSegment.builder()
-                                               .dataSource("foo")
-                                               .interval(schemaInfo.getDataInterval())
-                                               .version("1")
-                                               .shardSpec(new LinearShardSpec(1))
-                                               .size(0)
-                                               .build();
-
-
-    final PlannerConfig plannerConfig = new PlannerConfig();
-
-    String columnCardinalityWithUnderscores = groupingDimension.substring(groupingDimension.lastIndexOf('-') + 1);
-    int rowsPerSegment = Integer.parseInt(StringUtils.replace(columnCardinalityWithUnderscores, "_", ""));
-
-    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
-
-    TransformSpec transformSpec = new TransformSpec(
-        null,
-        ImmutableList.of(
-            // string array dims
-            new ExpressionTransform(
-                "stringArray-Sequential-100_000",
-                "array(\"string-Sequential-100_000\")",
-                TestExprMacroTable.INSTANCE
-            ),
-            new ExpressionTransform(
-                "stringArray-Sequential-3_000_000",
-                "array(\"string-Sequential-10_000_000\")",
-                TestExprMacroTable.INSTANCE
-            ),
-            /*
-            new ExpressionTransform(
-                "stringArray-Sequential-1_000_000_000",
-                "array(\"string-Sequential-1_000_000_000\")",
-                TestExprMacroTable.INSTANCE
-            ),*/
-            new ExpressionTransform(
-                "stringArray-ZipF-1_000_000",
-                "array(\"string-ZipF-1_000_000\")",
-                TestExprMacroTable.INSTANCE
-            ),
-            new ExpressionTransform(
-                "stringArray-Uniform-1_000_000",
-                "array(\"string-Uniform-1_000_000\")",
-                TestExprMacroTable.INSTANCE
-            ),
-
-            // long array dims
-            new ExpressionTransform(
-                "longArray-Sequential-100_000",
-                "array(\"long-Sequential-100_000\")",
-                TestExprMacroTable.INSTANCE
-            ),
-            new ExpressionTransform(
-                "longArray-Sequential-3_000_000",
-                "array(\"long-Sequential-10_000_000\")",
-                TestExprMacroTable.INSTANCE
-            ),
-            /*
-            new ExpressionTransform(
-                "longArray-Sequential-1_000_000_000",
-                "array(\"long-Sequential-1_000_000_000\")",
-                TestExprMacroTable.INSTANCE
-            ),*/
-            new ExpressionTransform(
-                "longArray-ZipF-1_000_000",
-                "array(\"long-ZipF-1_000_000\")",
-                TestExprMacroTable.INSTANCE
-            ),
-            new ExpressionTransform(
-                "longArray-Uniform-1_000_000",
-                "array(\"long-Uniform-1_000_000\")",
-                TestExprMacroTable.INSTANCE
-            ),
-
-            // nested complex json dim
-            new ExpressionTransform(
-                "nested-Sequential-100_000",
-                "json_object('long1', \"long-Sequential-100_000\", 'nesteder', json_object('long1', \"long-Sequential-100_000\"))",
-                TestExprMacroTable.INSTANCE
-            ),
-            new ExpressionTransform(
-                "nested-Sequential-3_000_000",
-                "json_object('long1', \"long-Sequential-10_000_000\", 'nesteder', json_object('long1', \"long-Sequential-10_000_000\"))",
-                TestExprMacroTable.INSTANCE
-            ),
-            /*new ExpressionTransform(
-                "nested-Sequential-1_000_000_000",
-                "json_object('long1', \"long-Sequential-1_000_000_000\", 'nesteder', json_object('long1', \"long-Sequential-1_000_000_000\"))",
-                TestExprMacroTable.INSTANCE
-            ),*/
-            new ExpressionTransform(
-                "nested-ZipF-1_000_000",
-                "json_object('long1', \"long-ZipF-1_000_000\", 'nesteder', json_object('long1', \"long-ZipF-1_000_000\"))",
-                TestExprMacroTable.INSTANCE
-            ),
-            new ExpressionTransform(
-                "nested-Uniform-1_000_000",
-                "json_object('long1', \"long-Uniform-1_000_000\", 'nesteder', json_object('long1', \"long-Uniform-1_000_000\"))",
-                TestExprMacroTable.INSTANCE
-            )
-        )
-    );
-
-    List<DimensionSchema> columnSchemas = schemaInfo.getDimensionsSpec()
-                                                    .getDimensions()
-                                                    .stream()
-                                                    .map(x -> new AutoTypeColumnSchema(x.getName(), null))
-                                                    .collect(Collectors.toList());
-
-    List<DimensionSchema> transformSchemas = transformSpec
-        .getTransforms()
-        .stream()
-        .map(
-            transform -> new AutoTypeColumnSchema(transform.getName(), null)
-        )
-        .collect(Collectors.toList());
-
-
-
-    final QueryableIndex index = segmentGenerator.generate(
-        dataSegment,
-        schemaInfo,
-        DimensionsSpec.builder()
-                      .setDimensions(ImmutableList.<DimensionSchema>builder()
-                                                  .addAll(columnSchemas)
-                                                  .addAll(transformSchemas)
-                                                  .build()
-                      )
-                      .build(),
-        transformSpec,
-        IndexSpec.builder().withStringDictionaryEncoding(new StringEncodingStrategy.Utf8()).build(),
-        Granularities.NONE,
-        rowsPerSegment
-    );
-
-    final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(
-        closer,
-        PROCESSING_CONFIG
-    );
-
-    final SpecificSegmentsQuerySegmentWalker walker = SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate)
-                                                                                        .add(dataSegment, index)
-                                                                                        .add(dataSegment2, index);
-    closer.register(walker);
-
-    // Hacky and pollutes global namespace, but it is fine since benchmarks are run in isolation. Wasn't able
-    // to work up a cleaner way of doing it by modifying the injector.
-    CalciteTests.getJsonMapper().registerModules(BuiltInTypesModule.getJacksonModulesList());
-
-    final DruidSchemaCatalog rootSchema =
-        CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
-    engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
-    plannerFactory = new PlannerFactory(
-        rootSchema,
-        CalciteTests.createOperatorTable(),
-        CalciteTests.createExprMacroTable(),
-        plannerConfig,
-        AuthTestUtils.TEST_AUTHORIZER_MAPPER,
-        CalciteTests.getJsonMapper(),
-        CalciteTests.DRUID_SCHEMA_NAME,
-        new CalciteRulesManager(ImmutableSet.of()),
-        CalciteTests.createJoinableFactoryWrapper(),
-        CatalogResolver.NULL_RESOLVER,
-        new AuthConfig(),
-        new DruidHookDispatcher()
-    );
-
-    try {
-      SqlVectorizedExpressionSanityTest.sanityTestVectorizedSqlQueries(
-          engine,
-          plannerFactory,
-          sqlQuery(groupingDimension)
-      );
-      log.info("non-vectorized and vectorized results match");
-    }
-    catch (Throwable ex) {
-      log.warn(ex, "non-vectorized and vectorized results do not match");
-    }
-  }
-
-  @TearDown(Level.Trial)
-  public void tearDown() throws Exception
-  {
-    closer.close();
-  }
 
-  @Benchmark
-  @BenchmarkMode(Mode.AverageTime)
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  public void querySql(Blackhole blackhole)
+  @Override
+  public String getQuery()
   {
-    final String sql = sqlQuery(groupingDimension);
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, Collections.emptyMap())) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
-      blackhole.consume(lastRow);
-    }
+    return StringUtils.format("SELECT \"%s\", COUNT(*) FROM druid.%s GROUP BY 1", groupingDimension, SqlBenchmarkDatasets.GROUPER);
   }
 
-  private static String sqlQuery(String groupingDimension)
+  @Override
+  public List<String> getDatasources()
   {
-    return StringUtils.format("SELECT \"%s\", COUNT(*) FROM foo GROUP BY 1", groupingDimension);
+    return Collections.singletonList(SqlBenchmarkDatasets.GROUPER);
   }
 }
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
index fa5942dc2c98..73fc97dd78a1 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java
@@ -19,224 +19,109 @@
 
 package org.apache.druid.benchmark.query;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Yielder;
-import org.apache.druid.java.util.common.guava.Yielders;
-import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.math.expr.ExpressionProcessing;
-import org.apache.druid.query.DruidProcessingConfig;
-import org.apache.druid.query.QueryContexts;
-import org.apache.druid.query.QueryRunnerFactoryConglomerate;
-import org.apache.druid.query.expression.TestExprMacroTable;
-import org.apache.druid.segment.AutoTypeColumnSchema;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.column.StringEncodingStrategy;
-import org.apache.druid.segment.data.FrontCodedIndexed;
-import org.apache.druid.segment.generator.GeneratorBasicSchemas;
-import org.apache.druid.segment.generator.GeneratorSchemaInfo;
-import org.apache.druid.segment.generator.SegmentGenerator;
-import org.apache.druid.segment.transform.ExpressionTransform;
-import org.apache.druid.segment.transform.TransformSpec;
-import org.apache.druid.server.QueryStackTests;
-import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
-import org.apache.druid.server.security.AuthConfig;
-import org.apache.druid.server.security.AuthTestUtils;
-import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest;
-import org.apache.druid.sql.calcite.planner.CalciteRulesManager;
-import org.apache.druid.sql.calcite.planner.CatalogResolver;
-import org.apache.druid.sql.calcite.planner.DruidPlanner;
-import org.apache.druid.sql.calcite.planner.PlannerConfig;
-import org.apache.druid.sql.calcite.planner.PlannerFactory;
-import org.apache.druid.sql.calcite.planner.PlannerResult;
-import org.apache.druid.sql.calcite.run.SqlEngine;
-import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
-import org.apache.druid.sql.calcite.util.CalciteTests;
-import org.apache.druid.sql.hook.DruidHookDispatcher;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.partition.LinearShardSpec;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
 
-import javax.annotation.Nullable;
-import java.util.Collections;
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 @State(Scope.Benchmark)
 @Fork(value = 1)
 @Warmup(iterations = 3)
 @Measurement(iterations = 5)
-public class SqlNestedDataBenchmark
+public class SqlNestedDataBenchmark extends SqlBaseQueryBenchmark
 {
-  private static final Logger log = new Logger(SqlNestedDataBenchmark.class);
-
-  static {
-    NullHandling.initializeForTests();
-    ExpressionProcessing.initializeForTests();
-  }
-
-  private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig()
-  {
-    @Override
-    public int intermediateComputeSizeBytes()
-    {
-      return 512 * 1024 * 1024;
-    }
-
-    @Override
-    public int getNumMergeBuffers()
-    {
-      return 3;
-    }
-
-    @Override
-    public int getNumThreads()
-    {
-      return 1;
-    }
-
-    @Override
-    public String getFormatString()
-    {
-      return "benchmarks-processing-%s";
-    }
-  };
-
-
   private static final List<String> QUERIES = ImmutableList.of(
       // ===========================
       // non-nested reference queries
       // ===========================
       // 0,1: timeseries, 1 columns
-      "SELECT SUM(long1) FROM foo",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo",
+      "SELECT SUM(long1) FROM druid.nested",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested",
       // 2,3: timeseries, 2 columns
-      "SELECT SUM(long1), SUM(long2) FROM foo",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)), SUM(JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT)) FROM foo",
+      "SELECT SUM(long1), SUM(long2) FROM druid.nested",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)), SUM(JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT)) FROM druid.nested",
       // 4,5: timeseries, 3 columns
-      "SELECT SUM(long1), SUM(long2), SUM(double3) FROM foo",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)), SUM(JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT)), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM foo",
+      "SELECT SUM(long1), SUM(long2), SUM(double3) FROM druid.nested",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)), SUM(JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT)), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM druid.nested",
       // 6,7: group by string with 1 agg
-      "SELECT string1, SUM(long1) FROM foo GROUP BY 1 ORDER BY 2",
-      "SELECT JSON_VALUE(nested, '$.nesteder.string1'), SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT string1, SUM(long1) FROM druid.nested GROUP BY 1 ORDER BY 2",
+      "SELECT JSON_VALUE(nested, '$.nesteder.string1'), SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested GROUP BY 1 ORDER BY 2",
       // 8,9: group by string with 2 agg
-      "SELECT string1, SUM(long1), SUM(double3) FROM foo GROUP BY 1 ORDER BY 2",
-      "SELECT JSON_VALUE(nested, '$.nesteder.string1'), SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM foo GROUP BY 1 ORDER BY 2",
+      "SELECT string1, SUM(long1), SUM(double3) FROM druid.nested GROUP BY 1 ORDER BY 2",
+      "SELECT JSON_VALUE(nested, '$.nesteder.string1'), SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM druid.nested GROUP BY 1 ORDER BY 2",
       // 10,11: time-series filter string
-      "SELECT SUM(long1) FROM foo WHERE string1 = '10000' OR string1 = '1000'",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string1') = '10000' OR JSON_VALUE(nested, '$.nesteder.string1') = '1000'",
+      "SELECT SUM(long1) FROM druid.nested WHERE string1 = '10000' OR string1 = '1000'",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string1') = '10000' OR JSON_VALUE(nested, '$.nesteder.string1') = '1000'",
       // 12,13: time-series filter long
-      "SELECT SUM(long1) FROM foo WHERE long2 = 10000 OR long2 = 1000",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) = 10000 OR JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) = 1000",
+      "SELECT SUM(long1) FROM druid.nested WHERE long2 = 10000 OR long2 = 1000",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) = 10000 OR JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) = 1000",
       // 14,15: time-series filter double
-      "SELECT SUM(long1) FROM foo WHERE double3 < 10000.0 AND double3 > 1000.0",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 10000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
+      "SELECT SUM(long1) FROM druid.nested WHERE double3 < 10000.0 AND double3 > 1000.0",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 10000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
       // 16,17: group by long filter by string
-      "SELECT long1, SUM(double3) FROM foo WHERE string1 = '10000' OR string1 = '1000' GROUP BY 1 ORDER BY 2",
-      "SELECT JSON_VALUE(nested, '$.long1' RETURNING BIGINT), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string1') = '10000' OR JSON_VALUE(nested, '$.nesteder.string1') = '1000' GROUP BY 1 ORDER BY 2",
+      "SELECT long1, SUM(double3) FROM druid.nested WHERE string1 = '10000' OR string1 = '1000' GROUP BY 1 ORDER BY 2",
+      "SELECT JSON_VALUE(nested, '$.long1' RETURNING BIGINT), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string1') = '10000' OR JSON_VALUE(nested, '$.nesteder.string1') = '1000' GROUP BY 1 ORDER BY 2",
       // 18,19: group by string filter by long
-      "SELECT string1, SUM(double3) FROM foo WHERE long2 < 10000 AND long2 > 1000 GROUP BY 1 ORDER BY 2",
-      "SELECT JSON_VALUE(nested, '$.nesteder.string1'), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) < 10000 AND JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) > 1000 GROUP BY 1 ORDER BY 2",
+      "SELECT string1, SUM(double3) FROM druid.nested WHERE long2 < 10000 AND long2 > 1000 GROUP BY 1 ORDER BY 2",
+      "SELECT JSON_VALUE(nested, '$.nesteder.string1'), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) < 10000 AND JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) > 1000 GROUP BY 1 ORDER BY 2",
       // 20,21: group by string filter by double
-      "SELECT string1, SUM(double3) FROM foo WHERE double3 < 10000.0 AND double3 > 1000.0 GROUP BY 1 ORDER BY 2",
-      "SELECT JSON_VALUE(nested, '$.nesteder.string1'), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 10000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0 GROUP BY 1 ORDER BY 2",
+      "SELECT string1, SUM(double3) FROM druid.nested WHERE double3 < 10000.0 AND double3 > 1000.0 GROUP BY 1 ORDER BY 2",
+      "SELECT JSON_VALUE(nested, '$.nesteder.string1'), SUM(JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 10000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0 GROUP BY 1 ORDER BY 2",
       // 22, 23:
-      "SELECT long2 FROM foo WHERE long2 IN (1, 19, 21, 23, 25, 26, 46)",
-      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46)",
+      "SELECT long2 FROM druid.nested WHERE long2 IN (1, 19, 21, 23, 25, 26, 46)",
+      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46)",
       // 24, 25
-      "SELECT long2 FROM foo WHERE long2 IN (1, 19, 21, 23, 25, 26, 46) GROUP BY 1",
-      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46) GROUP BY 1",
+      "SELECT long2 FROM druid.nested WHERE long2 IN (1, 19, 21, 23, 25, 26, 46) GROUP BY 1",
+      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46) GROUP BY 1",
       // 26, 27
-      "SELECT SUM(long1) FROM foo WHERE double3 < 1005.0 AND double3 > 1000.0",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 1005.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
+      "SELECT SUM(long1) FROM druid.nested WHERE double3 < 1005.0 AND double3 > 1000.0",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 1005.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
       // 28, 29
-      "SELECT SUM(long1) FROM foo WHERE double3 < 2000.0 AND double3 > 1000.0",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 2000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
+      "SELECT SUM(long1) FROM druid.nested WHERE double3 < 2000.0 AND double3 > 1000.0",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 2000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
       // 30, 31
-      "SELECT SUM(long1) FROM foo WHERE double3 < 3000.0 AND double3 > 1000.0",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 3000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
+      "SELECT SUM(long1) FROM druid.nested WHERE double3 < 3000.0 AND double3 > 1000.0",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 3000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
       // 32,33
-      "SELECT SUM(long1) FROM foo WHERE double3 < 5000.0 AND double3 > 1000.0",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 5000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
+      "SELECT SUM(long1) FROM druid.nested WHERE double3 < 5000.0 AND double3 > 1000.0",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 5000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0",
       // 34,35 smaller cardinality like range filter
-      "SELECT SUM(long1) FROM foo WHERE string1 LIKE '1%'",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string1') LIKE '1%'",
+      "SELECT SUM(long1) FROM druid.nested WHERE string1 LIKE '1%'",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string1') LIKE '1%'",
       // 36,37 smaller cardinality like predicate filter
-      "SELECT SUM(long1) FROM foo WHERE string1 LIKE '%1%'",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string1') LIKE '%1%'",
+      "SELECT SUM(long1) FROM druid.nested WHERE string1 LIKE '%1%'",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string1') LIKE '%1%'",
       // 38-39 moderate cardinality like range
-      "SELECT SUM(long1) FROM foo WHERE string5 LIKE '1%'",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '1%'",
+      "SELECT SUM(long1) FROM druid.nested WHERE string5 LIKE '1%'",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '1%'",
       // 40, 41 big cardinality lex range
-      "SELECT SUM(long1) FROM foo WHERE string5 > '1'",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string5') > '1'",
+      "SELECT SUM(long1) FROM druid.nested WHERE string5 > '1'",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string5') > '1'",
       // 42, 43 big cardinality like predicate filter
-      "SELECT SUM(long1) FROM foo WHERE string5 LIKE '%1%'",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '%1%'",
+      "SELECT SUM(long1) FROM druid.nested WHERE string5 LIKE '%1%'",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '%1%'",
       // 44, 45 big cardinality like filter + selector filter with different ordering
-      "SELECT SUM(long1) FROM foo WHERE string5 LIKE '%1%' AND string1 = '1000'",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '%1%' AND JSON_VALUE(nested, '$.nesteder.string1') = '1000'",
-      "SELECT SUM(long1) FROM foo WHERE string1 = '1000' AND string5 LIKE '%1%'",
-      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string1') = '1000' AND JSON_VALUE(nested, '$.nesteder.string5') LIKE '%1%'",
+      "SELECT SUM(long1) FROM druid.nested WHERE string5 LIKE '%1%' AND string1 = '1000'",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '%1%' AND JSON_VALUE(nested, '$.nesteder.string1') = '1000'",
+      "SELECT SUM(long1) FROM druid.nested WHERE string1 = '1000' AND string5 LIKE '%1%'",
+      "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.string1') = '1000' AND JSON_VALUE(nested, '$.nesteder.string5') LIKE '%1%'",
       //48,49 bigger in
-      "SELECT long2 FROM foo WHERE long2 IN (1, 19, 21, 23, 25, 26, 46, 50, 51, 55, 60, 61, 66, 68, 69, 70, 77, 88, 90, 92, 93, 94, 95, 100, 101, 102, 104, 109, 111, 113, 114, 115, 120, 121, 122, 134, 135, 136, 140, 142, 150, 155, 170, 172, 173, 174, 180, 181, 190, 199, 200, 201, 202, 203, 204)",
-      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46, 50, 51, 55, 60, 61, 66, 68, 69, 70, 77, 88, 90, 92, 93, 94, 95, 100, 101, 102, 104, 109, 111, 113, 114, 115, 120, 121, 122, 134, 135, 136, 140, 142, 150, 155, 170, 172, 173, 174, 180, 181, 190, 199, 200, 201, 202, 203, 204)",
+      "SELECT long2 FROM druid.nested WHERE long2 IN (1, 19, 21, 23, 25, 26, 46, 50, 51, 55, 60, 61, 66, 68, 69, 70, 77, 88, 90, 92, 93, 94, 95, 100, 101, 102, 104, 109, 111, 113, 114, 115, 120, 121, 122, 134, 135, 136, 140, 142, 150, 155, 170, 172, 173, 174, 180, 181, 190, 199, 200, 201, 202, 203, 204)",
+      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46, 50, 51, 55, 60, 61, 66, 68, 69, 70, 77, 88, 90, 92, 93, 94, 95, 100, 101, 102, 104, 109, 111, 113, 114, 115, 120, 121, 122, 134, 135, 136, 140, 142, 150, 155, 170, 172, 173, 174, 180, 181, 190, 199, 200, 201, 202, 203, 204)",
       //50, 51 bigger in group
-      "SELECT long2 FROM foo WHERE long2 IN (1, 19, 21, 23, 25, 26, 46, 50, 51, 55, 60, 61, 66, 68, 69, 70, 77, 88, 90, 92, 93, 94, 95, 100, 101, 102, 104, 109, 111, 113, 114, 115, 120, 121, 122, 134, 135, 136, 140, 142, 150, 155, 170, 172, 173, 174, 180, 181, 190, 199, 200, 201, 202, 203, 204) GROUP BY 1",
-      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46, 50, 51, 55, 60, 61, 66, 68, 69, 70, 77, 88, 90, 92, 93, 94, 95, 100, 101, 102, 104, 109, 111, 113, 114, 115, 120, 121, 122, 134, 135, 136, 140, 142, 150, 155, 170, 172, 173, 174, 180, 181, 190, 199, 200, 201, 202, 203, 204) GROUP BY 1",
-      "SELECT long2 FROM foo WHERE double3 IN (1.0, 19.0, 21.0, 23.0, 25.0, 26.0, 46.0, 50.0, 51.0, 55.0, 60.0, 61.0, 66.0, 68.0, 69.0, 70.0, 77.0, 88.0, 90.0, 92.0, 93.0, 94.0, 95.0, 100.0, 101.0, 102.0, 104.0, 109.0, 111.0, 113.0, 114.0, 115.0, 120.0, 121.0, 122.0, 134.0, 135.0, 136.0, 140.0, 142.0, 150.0, 155.0, 170.0, 172.0, 173.0, 174.0, 180.0, 181.0, 190.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0)",
-      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) IN (1.0, 19.0, 21.0, 23.0, 25.0, 26.0, 46.0, 50.0, 51.0, 55.0, 60.0, 61.0, 66.0, 68.0, 69.0, 70.0, 77.0, 88.0, 90.0, 92.0, 93.0, 94.0, 95.0, 100.0, 101.0, 102.0, 104.0, 109.0, 111.0, 113.0, 114.0, 115.0, 120.0, 121.0, 122.0, 134.0, 135.0, 136.0, 140.0, 142.0, 150.0, 155.0, 170.0, 172.0, 173.0, 174.0, 180.0, 181.0, 190.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0)",
-      "SELECT long2 FROM foo WHERE double3 IN (1.0, 19.0, 21.0, 23.0, 25.0, 26.0, 46.0, 50.0, 51.0, 55.0, 60.0, 61.0, 66.0, 68.0, 69.0, 70.0, 77.0, 88.0, 90.0, 92.0, 93.0, 94.0, 95.0, 100.0, 101.0, 102.0, 104.0, 109.0, 111.0, 113.0, 114.0, 115.0, 120.0, 121.0, 122.0, 134.0, 135.0, 136.0, 140.0, 142.0, 150.0, 155.0, 170.0, 172.0, 173.0, 174.0, 180.0, 181.0, 190.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0) GROUP BY 1",
-      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) IN (1.0, 19.0, 21.0, 23.0, 25.0, 26.0, 46.0, 50.0, 51.0, 55.0, 60.0, 61.0, 66.0, 68.0, 69.0, 70.0, 77.0, 88.0, 90.0, 92.0, 93.0, 94.0, 95.0, 100.0, 101.0, 102.0, 104.0, 109.0, 111.0, 113.0, 114.0, 115.0, 120.0, 121.0, 122.0, 134.0, 135.0, 136.0, 140.0, 142.0, 150.0, 155.0, 170.0, 172.0, 173.0, 174.0, 180.0, 181.0, 190.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0) GROUP BY 1"
+      "SELECT long2 FROM druid.nested WHERE long2 IN (1, 19, 21, 23, 25, 26, 46, 50, 51, 55, 60, 61, 66, 68, 69, 70, 77, 88, 90, 92, 93, 94, 95, 100, 101, 102, 104, 109, 111, 113, 114, 115, 120, 121, 122, 134, 135, 136, 140, 142, 150, 155, 170, 172, 173, 174, 180, 181, 190, 199, 200, 201, 202, 203, 204) GROUP BY 1",
+      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) IN (1, 19, 21, 23, 25, 26, 46, 50, 51, 55, 60, 61, 66, 68, 69, 70, 77, 88, 90, 92, 93, 94, 95, 100, 101, 102, 104, 109, 111, 113, 114, 115, 120, 121, 122, 134, 135, 136, 140, 142, 150, 155, 170, 172, 173, 174, 180, 181, 190, 199, 200, 201, 202, 203, 204) GROUP BY 1",
+      "SELECT long2 FROM druid.nested WHERE double3 IN (1.0, 19.0, 21.0, 23.0, 25.0, 26.0, 46.0, 50.0, 51.0, 55.0, 60.0, 61.0, 66.0, 68.0, 69.0, 70.0, 77.0, 88.0, 90.0, 92.0, 93.0, 94.0, 95.0, 100.0, 101.0, 102.0, 104.0, 109.0, 111.0, 113.0, 114.0, 115.0, 120.0, 121.0, 122.0, 134.0, 135.0, 136.0, 140.0, 142.0, 150.0, 155.0, 170.0, 172.0, 173.0, 174.0, 180.0, 181.0, 190.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0)",
+      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) IN (1.0, 19.0, 21.0, 23.0, 25.0, 26.0, 46.0, 50.0, 51.0, 55.0, 60.0, 61.0, 66.0, 68.0, 69.0, 70.0, 77.0, 88.0, 90.0, 92.0, 93.0, 94.0, 95.0, 100.0, 101.0, 102.0, 104.0, 109.0, 111.0, 113.0, 114.0, 115.0, 120.0, 121.0, 122.0, 134.0, 135.0, 136.0, 140.0, 142.0, 150.0, 155.0, 170.0, 172.0, 173.0, 174.0, 180.0, 181.0, 190.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0)",
+      "SELECT long2 FROM druid.nested WHERE double3 IN (1.0, 19.0, 21.0, 23.0, 25.0, 26.0, 46.0, 50.0, 51.0, 55.0, 60.0, 61.0, 66.0, 68.0, 69.0, 70.0, 77.0, 88.0, 90.0, 92.0, 93.0, 94.0, 95.0, 100.0, 101.0, 102.0, 104.0, 109.0, 111.0, 113.0, 114.0, 115.0, 120.0, 121.0, 122.0, 134.0, 135.0, 136.0, 140.0, 142.0, 150.0, 155.0, 170.0, 172.0, 173.0, 174.0, 180.0, 181.0, 190.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0) GROUP BY 1",
+      "SELECT JSON_VALUE(nested, '$.nesteder.long2' RETURNING BIGINT) FROM druid.nested WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) IN (1.0, 19.0, 21.0, 23.0, 25.0, 26.0, 46.0, 50.0, 51.0, 55.0, 60.0, 61.0, 66.0, 68.0, 69.0, 70.0, 77.0, 88.0, 90.0, 92.0, 93.0, 94.0, 95.0, 100.0, 101.0, 102.0, 104.0, 109.0, 111.0, 113.0, 114.0, 115.0, 120.0, 121.0, 122.0, 134.0, 135.0, 136.0, 140.0, 142.0, 150.0, 155.0, 170.0, 172.0, 173.0, 174.0, 180.0, 181.0, 190.0, 199.0, 200.0, 201.0, 202.0, 203.0, 204.0) GROUP BY 1"
   );
 
-  @Param({"5000000"})
-  private int rowsPerSegment;
-
-  @Param({
-      "false",
-      "force"
-  })
-  private String vectorize;
-
-  @Param({
-      "none",
-      "front-coded-4",
-      "front-coded-16"
-  })
-  private String stringEncoding;
-
-  @Param({
-      "explicit",
-      "auto"
-  })
-  private String schema;
 
   @Param({
       "0",
@@ -298,179 +183,15 @@ public String getFormatString()
   })
   private String query;
 
-  private SqlEngine engine;
-  @Nullable
-  private PlannerFactory plannerFactory;
-  private final Closer closer = Closer.create();
-
-  @Setup(Level.Trial)
-  public void setup()
-  {
-    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("expression-testbench");
-
-    final DataSegment dataSegment = DataSegment.builder()
-                                               .dataSource("foo")
-                                               .interval(schemaInfo.getDataInterval())
-                                               .version("1")
-                                               .shardSpec(new LinearShardSpec(0))
-                                               .size(0)
-                                               .build();
-
-
-    final PlannerConfig plannerConfig = new PlannerConfig();
-
-    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
-    log.info("Starting benchmark setup using cacheDir[%s], rows[%,d].", segmentGenerator.getCacheDir(), rowsPerSegment);
-
-    TransformSpec transformSpec = new TransformSpec(
-        null,
-        ImmutableList.of(
-            new ExpressionTransform(
-                "nested",
-                "json_object('long1', long1, 'nesteder', json_object('string1', string1, 'long2', long2, 'double3',double3, 'string5', string5))",
-                TestExprMacroTable.INSTANCE
-            )
-        )
-    );
-
-
-
-    StringEncodingStrategy encodingStrategy;
-    if (stringEncoding.startsWith("front-coded")) {
-      String[] split = stringEncoding.split("-");
-      int bucketSize = Integer.parseInt(split[2]);
-      encodingStrategy = new StringEncodingStrategy.FrontCoded(bucketSize, FrontCodedIndexed.V1);
-    } else {
-      encodingStrategy = new StringEncodingStrategy.Utf8();
-    }
-    final QueryableIndex index;
-    if ("auto".equals(schema)) {
-      Iterable<DimensionSchema> columnSchemas = Iterables.concat(
-          schemaInfo.getDimensionsSpec()
-                    .getDimensions()
-                    .stream()
-                    .map(x -> new AutoTypeColumnSchema(x.getName(), null))
-                    .collect(Collectors.toList()),
-          Collections.singletonList(new AutoTypeColumnSchema("nested", null))
-      );
-      index = segmentGenerator.generate(
-          dataSegment,
-          schemaInfo,
-          DimensionsSpec.builder().setDimensions(ImmutableList.copyOf(columnSchemas.iterator())).build(),
-          transformSpec,
-          IndexSpec.builder().withStringDictionaryEncoding(encodingStrategy).build(),
-          Granularities.NONE,
-          rowsPerSegment
-      );
-    } else {
-      Iterable<DimensionSchema> columnSchemas = Iterables.concat(
-          schemaInfo.getDimensionsSpec().getDimensions(),
-          Collections.singletonList(new AutoTypeColumnSchema("nested", null))
-      );
-      index = segmentGenerator.generate(
-          dataSegment,
-          schemaInfo,
-          DimensionsSpec.builder().setDimensions(ImmutableList.copyOf(columnSchemas.iterator())).build(),
-          transformSpec,
-          IndexSpec.builder().withStringDictionaryEncoding(encodingStrategy).build(),
-          Granularities.NONE,
-          rowsPerSegment
-      );
-    }
-
-    final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(
-        closer,
-        PROCESSING_CONFIG
-    );
-
-    final SpecificSegmentsQuerySegmentWalker walker = SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate).add(
-        dataSegment,
-        index
-    );
-    closer.register(walker);
-
-    final DruidSchemaCatalog rootSchema =
-        CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
-    engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
-    plannerFactory = new PlannerFactory(
-        rootSchema,
-        CalciteTests.createOperatorTable(),
-        CalciteTests.createExprMacroTable(),
-        plannerConfig,
-        AuthTestUtils.TEST_AUTHORIZER_MAPPER,
-        CalciteTests.getJsonMapper(),
-        CalciteTests.DRUID_SCHEMA_NAME,
-        new CalciteRulesManager(ImmutableSet.of()),
-        CalciteTests.createJoinableFactoryWrapper(),
-        CatalogResolver.NULL_RESOLVER,
-        new AuthConfig(),
-        new DruidHookDispatcher()
-    );
-
-    try {
-      SqlVectorizedExpressionSanityTest.sanityTestVectorizedSqlQueries(
-          engine,
-          plannerFactory,
-          QUERIES.get(Integer.parseInt(query))
-      );
-      log.info("non-vectorized and vectorized results match");
-    }
-    catch (Throwable ex) {
-      log.warn(ex, "non-vectorized and vectorized results do not match");
-    }
-
-    final String sql = QUERIES.get(Integer.parseInt(query));
-    final ObjectMapper jsonMapper = CalciteTests.getJsonMapper();
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, "EXPLAIN PLAN FOR " + sql, ImmutableMap.of("useNativeQueryExplain", true))) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Object[] planResult = resultSequence.toList().get(0);
-      log.info("Native query plan:\n" +
-               jsonMapper.writerWithDefaultPrettyPrinter()
-                         .writeValueAsString(jsonMapper.readValue((String) planResult[0], List.class))
-      );
-    }
-    catch (JsonProcessingException ex) {
-      log.warn(ex, "explain failed");
-    }
-
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, ImmutableMap.of())) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Yielder<Object[]> yielder = Yielders.each(resultSequence);
-      int rowCounter = 0;
-      while (!yielder.isDone()) {
-        rowCounter++;
-        yielder.next(yielder.get());
-      }
-      log.info("Total result row count:" + rowCounter);
-    }
-    catch (Throwable ex) {
-      log.warn(ex, "failed to count rows");
-    }
-  }
-
-  @TearDown(Level.Trial)
-  public void tearDown() throws Exception
+  @Override
+  public String getQuery()
   {
-    closer.close();
+    return QUERIES.get(Integer.parseInt(query));
   }
 
-  @Benchmark
-  @BenchmarkMode(Mode.AverageTime)
-  @OutputTimeUnit(TimeUnit.MILLISECONDS)
-  public void querySql(Blackhole blackhole)
+  @Override
+  public List<String> getDatasources()
   {
-    final Map<String, Object> context = ImmutableMap.of(
-        QueryContexts.VECTORIZE_KEY, vectorize,
-        QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
-    );
-    final String sql = QUERIES.get(Integer.parseInt(query));
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, context)) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
-      blackhole.consume(lastRow);
-    }
+    return ImmutableList.of(SqlBenchmarkDatasets.NESTED);
   }
 }
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlPlanBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlPlanBenchmark.java
new file mode 100644
index 000000000000..29e9cb2c24ff
--- /dev/null
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlPlanBenchmark.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableList;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.List;
+
+@State(Scope.Benchmark)
+@Fork(value = 1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+public class SqlPlanBenchmark extends SqlBasePlanBenchmark
+{
+  @Param({
+      "0",
+      "1",
+      "2",
+      "3",
+      "4",
+      "5",
+      "6",
+      "7",
+      "8",
+      "9",
+      "10",
+      "11",
+      "12",
+      "13",
+      "14",
+      "15",
+      "16",
+      "17",
+      "18",
+      "19",
+      "20",
+      "21",
+      "22",
+      "23",
+      "24",
+      "25",
+      "26",
+      "27",
+      "28",
+      "29",
+      "30",
+      "31",
+      "32",
+      "33",
+      "34",
+      "35",
+      "36",
+      "37",
+      "38",
+      "39",
+      "40",
+      "41"
+  })
+  private String query;
+
+  @Override
+  public String getQuery()
+  {
+    return SqlBenchmark.QUERIES.get(Integer.parseInt(query));
+  }
+
+  @Override
+  public List<String> getDatasources()
+  {
+    return ImmutableList.of(SqlBenchmarkDatasets.BASIC);
+  }
+}
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlProjectionsBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlProjectionsBenchmark.java
new file mode 100644
index 000000000000..78b56eadf365
--- /dev/null
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlProjectionsBenchmark.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.benchmark.query;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.druid.query.QueryContexts;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.List;
+import java.util.Map;
+
+@State(Scope.Benchmark)
+@Fork(value = 1)
+@Warmup(iterations = 3)
+@Measurement(iterations = 5)
+public class SqlProjectionsBenchmark extends SqlBaseQueryBenchmark
+{
+  private static final List<String> QUERIES = ImmutableList.of(
+      "SELECT string2, APPROX_COUNT_DISTINCT_DS_HLL(string5) FROM druid.projections GROUP BY 1 ORDER BY 2",
+      "SELECT string2, SUM(long4) FROM druid.projections GROUP BY 1 ORDER BY 2"
+  );
+
+  @Param({
+      "0",
+      "1"
+  })
+  private String query;
+
+  @Param({
+      "true",
+      "false"
+  })
+  private boolean useProjections;
+
+  @Override
+  public String getQuery()
+  {
+    return QUERIES.get(Integer.parseInt(query));
+  }
+
+  @Override
+  public List<String> getDatasources()
+  {
+    return ImmutableList.of(SqlBenchmarkDatasets.PROJECTIONS);
+  }
+
+  @Override
+  protected Map<String, Object> getContext()
+  {
+    final Map<String, Object> context = ImmutableMap.of(
+        QueryContexts.VECTORIZE_KEY, vectorize,
+        QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize,
+        useProjections ? QueryContexts.FORCE_PROJECTION : QueryContexts.NO_PROJECTIONS, true
+    );
+    return context;
+  }
+}
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlWindowFunctionsBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlWindowFunctionsBenchmark.java
index b5398ead4c9f..550dad624f19 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlWindowFunctionsBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlWindowFunctionsBenchmark.java
@@ -19,98 +19,22 @@
 
 package org.apache.druid.benchmark.query;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import com.google.inject.multibindings.MapBinder;
-import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.frame.FrameType;
-import org.apache.druid.frame.testutil.FrameTestUtil;
-import org.apache.druid.guice.ExpressionModule;
-import org.apache.druid.guice.LazySingleton;
-import org.apache.druid.guice.SegmentWranglerModule;
-import org.apache.druid.guice.StartupInjectorBuilder;
-import org.apache.druid.guice.annotations.Json;
-import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.io.Closer;
-import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.math.expr.ExprMacroTable;
-import org.apache.druid.query.DruidProcessingConfig;
 import org.apache.druid.query.QueryContexts;
-import org.apache.druid.query.QueryRunnerFactoryConglomerate;
-import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctSqlAggregator;
-import org.apache.druid.query.aggregation.datasketches.hll.sql.HllSketchApproxCountDistinctUtf8SqlAggregator;
-import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchApproxQuantileSqlAggregator;
-import org.apache.druid.query.aggregation.datasketches.quantiles.sql.DoublesSketchObjectSqlAggregator;
-import org.apache.druid.query.aggregation.datasketches.theta.sql.ThetaSketchApproxCountDistinctSqlAggregator;
-import org.apache.druid.query.lookup.LookupExtractor;
-import org.apache.druid.segment.AutoTypeColumnSchema;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.QueryableIndex;
-import org.apache.druid.segment.QueryableIndexCursorFactory;
-import org.apache.druid.segment.QueryableIndexSegment;
-import org.apache.druid.segment.column.StringEncodingStrategy;
-import org.apache.druid.segment.generator.GeneratorBasicSchemas;
-import org.apache.druid.segment.generator.GeneratorSchemaInfo;
-import org.apache.druid.segment.generator.SegmentGenerator;
-import org.apache.druid.segment.join.JoinableFactoryWrapper;
-import org.apache.druid.segment.transform.TransformSpec;
-import org.apache.druid.server.QueryStackTests;
-import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
-import org.apache.druid.server.security.AuthConfig;
-import org.apache.druid.server.security.AuthTestUtils;
-import org.apache.druid.sql.calcite.aggregation.ApproxCountDistinctSqlAggregator;
-import org.apache.druid.sql.calcite.aggregation.SqlAggregationModule;
-import org.apache.druid.sql.calcite.aggregation.SqlAggregator;
-import org.apache.druid.sql.calcite.aggregation.builtin.CountSqlAggregator;
-import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
-import org.apache.druid.sql.calcite.expression.builtin.QueryLookupOperatorConversion;
-import org.apache.druid.sql.calcite.planner.CalciteRulesManager;
-import org.apache.druid.sql.calcite.planner.CatalogResolver;
-import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
-import org.apache.druid.sql.calcite.planner.DruidPlanner;
-import org.apache.druid.sql.calcite.planner.PlannerConfig;
-import org.apache.druid.sql.calcite.planner.PlannerFactory;
-import org.apache.druid.sql.calcite.planner.PlannerResult;
-import org.apache.druid.sql.calcite.run.SqlEngine;
-import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog;
-import org.apache.druid.sql.calcite.util.CalciteTests;
-import org.apache.druid.sql.calcite.util.LookylooModule;
-import org.apache.druid.sql.calcite.util.QueryFrameworkUtils;
-import org.apache.druid.sql.calcite.util.testoperator.CalciteTestOperatorModule;
-import org.apache.druid.sql.hook.DruidHookDispatcher;
-import org.apache.druid.timeline.DataSegment;
-import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Level;
 import org.openjdk.jmh.annotations.Measurement;
 import org.openjdk.jmh.annotations.Mode;
 import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Param;
 import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
 import org.openjdk.jmh.annotations.Warmup;
-import org.openjdk.jmh.infra.Blackhole;
 
-import javax.annotation.Nullable;
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 /**
  * Benchmark that tests various SQL queries.
@@ -121,319 +45,55 @@
 @Fork(value = 1)
 @Warmup(iterations = 3)
 @Measurement(iterations = 5)
-public class SqlWindowFunctionsBenchmark
+public class SqlWindowFunctionsBenchmark extends SqlBaseQueryBenchmark
 {
-  static {
-    NullHandling.initializeForTests();
-  }
-
-  private static final Logger log = new Logger(SqlWindowFunctionsBenchmark.class);
+  private static final List<String> QUERIES = ImmutableList.of(
+      "SELECT SUM(dimSequentialHalfNull) FROM druid.basic GROUP BY dimUniform",
+      "SELECT SUM(SUM(dimSequentialHalfNull)) OVER (ORDER BY dimUniform) FROM druid.basic GROUP BY dimUniform",
+      "SELECT ROW_NUMBER() OVER (PARTITION BY dimUniform ORDER BY dimSequential) FROM druid.basic",
+      "SELECT COUNT(*) OVER (PARTITION BY dimUniform RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) FROM druid.basic",
+      "SELECT COUNT(*) OVER (PARTITION BY dimUniform ORDER BY dimSequential RANGE UNBOUNDED PRECEDING) FROM druid.basic",
+      "SELECT COUNT(*) OVER (PARTITION BY dimUniform ORDER BY dimSequential RANGE UNBOUNDED FOLLOWING) FROM druid.basic",
+      "SELECT COUNT(*) OVER (PARTITION BY dimUniform ORDER BY dimSequential) FROM druid.basic GROUP BY dimSequential, dimUniform",
+      "SELECT COUNT(*) OVER (PARTITION BY dimUniform ORDER BY dimSequential) FROM druid.basic GROUP BY dimUniform, dimSequential",
+      "SELECT SUM(dimSequentialHalfNull) + SUM(dimZipf), LAG(SUM(dimSequentialHalfNull + dimZipf)) OVER (PARTITION BY dimUniform ORDER BY dimSequential) FROM druid.basic GROUP BY __time, dimUniform, dimSequential"
+  );
 
-  private static final String STORAGE_MMAP = "mmap";
-  private static final String STORAGE_FRAME_ROW = "frame-row";
-  private static final String STORAGE_FRAME_COLUMNAR = "frame-columnar";
-
-  @Param({"2000000"})
-  private int rowsPerSegment;
 
   @Param({
-      "auto"
+      "0",
+      "1",
+      "2",
+      "3",
+      "4",
+      "5",
+      "6",
+      "7",
+      "8"
   })
-  private String schema;
-
-  // Can be STORAGE_MMAP, STORAGE_FRAME_ROW, or STORAGE_FRAME_COLUMNAR
-  @Param({STORAGE_MMAP})
-  private String storageType;
-
-  private SqlEngine engine;
-
-  @Nullable
-  private PlannerFactory plannerFactory;
-  private final Closer closer = Closer.create();
-
-  private static final DruidProcessingConfig PROCESSING_CONFIG = new DruidProcessingConfig() {
-
-    @Override
-    public int getNumMergeBuffers()
-    {
-      return 3;
-    }
-
-    @Override
-    public int intermediateComputeSizeBytes()
-    {
-      return 200_000_000;
-    }
-  };
-
-  @Setup(Level.Trial)
-  public void setup()
-  {
-    final GeneratorSchemaInfo schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get("basic");
-    final DataSegment dataSegment = schemaInfo.makeSegmentDescriptor("foo");
-    final SegmentGenerator segmentGenerator = closer.register(new SegmentGenerator());
-
-    log.info("Starting benchmark setup using cacheDir[%s], rows[%,d].", segmentGenerator.getCacheDir(), rowsPerSegment);
-    final QueryableIndex index;
-    if ("auto".equals(schema)) {
-      List<DimensionSchema> columnSchemas = schemaInfo.getDimensionsSpec()
-                                                      .getDimensions()
-                                                      .stream()
-                                                      .map(x -> new AutoTypeColumnSchema(x.getName(), null))
-                                                      .collect(Collectors.toList());
-      index = segmentGenerator.generate(
-          dataSegment,
-          schemaInfo,
-          DimensionsSpec.builder().setDimensions(columnSchemas).build(),
-          TransformSpec.NONE,
-          IndexSpec.builder().withStringDictionaryEncoding(getStringEncodingStrategy()).build(),
-          Granularities.NONE,
-          rowsPerSegment
-      );
-    } else {
-      index = segmentGenerator.generate(dataSegment, schemaInfo, Granularities.NONE, rowsPerSegment);
-    }
-
-    final Pair<PlannerFactory, SqlEngine> sqlSystem = createSqlSystem(
-        ImmutableMap.of(dataSegment, index),
-        Collections.emptyMap(),
-        null,
-        closer
-    );
-
-    plannerFactory = sqlSystem.lhs;
-    engine = sqlSystem.rhs;
-  }
-
-  private StringEncodingStrategy getStringEncodingStrategy()
-  {
-    return new StringEncodingStrategy.Utf8();
-  }
-
-  public static Pair<PlannerFactory, SqlEngine> createSqlSystem(
-      final Map<DataSegment, QueryableIndex> segmentMap,
-      final Map<String, LookupExtractor> lookupMap,
-      @Nullable final String storageType,
-      final Closer closer
-  )
-  {
-    final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(closer, PROCESSING_CONFIG);
-    final SpecificSegmentsQuerySegmentWalker walker = SpecificSegmentsQuerySegmentWalker.createWalker(conglomerate);
-    final PlannerConfig plannerConfig = new PlannerConfig();
-
-    for (final Map.Entry<DataSegment, QueryableIndex> segmentEntry : segmentMap.entrySet()) {
-      addSegmentToWalker(walker, segmentEntry.getKey(), segmentEntry.getValue(), storageType);
-    }
-
-    // Child injector that adds additional lookups.
-    final Injector injector = new StartupInjectorBuilder()
-        .withEmptyProperties()
-        .add(
-            new ExpressionModule(),
-            new SegmentWranglerModule(),
-            new LookylooModule(),
-            new SqlAggregationModule(),
-            new CalciteTestOperatorModule(),
-            binder -> {
-              for (Map.Entry<String, LookupExtractor> entry : lookupMap.entrySet()) {
-                MapBinder.newMapBinder(binder, String.class, LookupExtractor.class)
-                         .addBinding(entry.getKey())
-                         .toProvider(entry::getValue)
-                         .in(LazySingleton.class);
-              }
-            }
-        )
-        .build();
-
-    final DruidSchemaCatalog rootSchema =
-        QueryFrameworkUtils.createMockRootSchema(
-            injector,
-            conglomerate,
-            walker,
-            plannerConfig,
-            AuthTestUtils.TEST_AUTHORIZER_MAPPER
-        );
-
-    final SqlEngine engine = CalciteTests.createMockSqlEngine(walker, conglomerate);
-
-    final PlannerFactory plannerFactory = new PlannerFactory(
-        rootSchema,
-        createOperatorTable(injector),
-        injector.getInstance(ExprMacroTable.class),
-        plannerConfig,
-        AuthTestUtils.TEST_AUTHORIZER_MAPPER,
-        injector.getInstance(Key.get(ObjectMapper.class, Json.class)),
-        CalciteTests.DRUID_SCHEMA_NAME,
-        new CalciteRulesManager(ImmutableSet.of()),
-        new JoinableFactoryWrapper(QueryFrameworkUtils.createDefaultJoinableFactory(injector)),
-        CatalogResolver.NULL_RESOLVER,
-        new AuthConfig(),
-        new DruidHookDispatcher()
-    );
-
-    return Pair.of(plannerFactory, engine);
-  }
+  private String query;
 
-  private static void addSegmentToWalker(
-      final SpecificSegmentsQuerySegmentWalker walker,
-      final DataSegment descriptor,
-      final QueryableIndex index,
-      @Nullable final String storageType
-  )
-  {
-    if (storageType == null || STORAGE_MMAP.equals(storageType)) {
-      walker.add(descriptor, new QueryableIndexSegment(index, descriptor.getId()));
-    } else if (STORAGE_FRAME_ROW.equals(storageType)) {
-      walker.add(
-          descriptor,
-          FrameTestUtil.cursorFactoryToFrameSegment(
-              new QueryableIndexCursorFactory(index),
-              FrameType.ROW_BASED,
-              descriptor.getId()
-          )
-      );
-    } else if (STORAGE_FRAME_COLUMNAR.equals(storageType)) {
-      walker.add(
-          descriptor,
-          FrameTestUtil.cursorFactoryToFrameSegment(
-              new QueryableIndexCursorFactory(index),
-              FrameType.COLUMNAR,
-              descriptor.getId()
-          )
-      );
-    } else {
-      throw new IAE("Invalid storageType[%s]", storageType);
-    }
-  }
 
-  private static DruidOperatorTable createOperatorTable(final Injector injector)
+  @Override
+  public String getQuery()
   {
-    try {
-      final Set<SqlOperatorConversion> extractionOperators = new HashSet<>();
-      extractionOperators.add(injector.getInstance(QueryLookupOperatorConversion.class));
-      final ApproxCountDistinctSqlAggregator countDistinctSqlAggregator =
-          new ApproxCountDistinctSqlAggregator(new HllSketchApproxCountDistinctSqlAggregator());
-      final Set<SqlAggregator> aggregators = new HashSet<>(
-          ImmutableList.of(
-              new DoublesSketchApproxQuantileSqlAggregator(),
-              new DoublesSketchObjectSqlAggregator(),
-              new HllSketchApproxCountDistinctSqlAggregator(),
-              new HllSketchApproxCountDistinctUtf8SqlAggregator(),
-              new ThetaSketchApproxCountDistinctSqlAggregator(),
-              new CountSqlAggregator(countDistinctSqlAggregator),
-              countDistinctSqlAggregator
-          )
-      );
-      return new DruidOperatorTable(aggregators, extractionOperators);
-    }
-    catch (Exception e) {
-      throw new RuntimeException(e);
-    }
+    return QUERIES.get(Integer.parseInt(query));
   }
 
-  @TearDown(Level.Trial)
-  public void tearDown() throws Exception
+  @Override
+  public List<String> getDatasources()
   {
-    closer.close();
+    return ImmutableList.of(SqlBenchmarkDatasets.BASIC);
   }
 
-  public void querySql(String sql, Blackhole blackhole)
+  @Override
+  protected Map<String, Object> getContext()
   {
-    final Map<String, Object> context = ImmutableMap.of(
+    return ImmutableMap.of(
         QueryContexts.MAX_SUBQUERY_BYTES_KEY, "disabled",
-        QueryContexts.MAX_SUBQUERY_ROWS_KEY, -1
+        QueryContexts.MAX_SUBQUERY_ROWS_KEY, -1,
+        QueryContexts.VECTORIZE_KEY, vectorize,
+        QueryContexts.VECTORIZE_VIRTUAL_COLUMNS_KEY, vectorize
     );
-    try (final DruidPlanner planner = plannerFactory.createPlannerForTesting(engine, sql, context)) {
-      final PlannerResult plannerResult = planner.plan();
-      final Sequence<Object[]> resultSequence = plannerResult.run().getResults();
-      final Object[] lastRow = resultSequence.accumulate(null, (accumulated, in) -> in);
-      blackhole.consume(lastRow);
-    }
-  }
-
-  @Benchmark
-  public void groupByWithoutWindow(Blackhole blackhole)
-  {
-    String sql = "SELECT SUM(dimSequentialHalfNull) "
-                 + "FROM foo "
-                 + "GROUP BY dimUniform";
-    querySql(sql, blackhole);
-  }
-
-  @Benchmark
-  public void groupByWithWindow(Blackhole blackhole)
-  {
-    String sql = "SELECT SUM(SUM(dimSequentialHalfNull)) "
-                 + "OVER (ORDER BY dimUniform) "
-                 + "FROM foo "
-                 + "GROUP BY dimUniform";
-    querySql(sql, blackhole);
-  }
-
-  @Benchmark
-  public void simpleWindow(Blackhole blackhole)
-  {
-    String sql = "SELECT ROW_NUMBER() "
-                 + "OVER (PARTITION BY dimUniform ORDER BY dimSequential) "
-                 + "FROM foo";
-    querySql(sql, blackhole);
-  }
-
-  @Benchmark
-  public void simpleWindowUnbounded(Blackhole blackhole)
-  {
-    String sql = "SELECT COUNT(*) "
-                 + "OVER (PARTITION BY dimUniform RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) "
-                 + "FROM foo";
-    querySql(sql, blackhole);
-  }
-
-  @Benchmark
-  public void windowTillCurrentRow(Blackhole blackhole)
-  {
-    String sql = "SELECT COUNT(*) "
-                 + "OVER (PARTITION BY dimUniform ORDER BY dimSequential RANGE UNBOUNDED PRECEDING) "
-                 + "FROM foo";
-    querySql(sql, blackhole);
-  }
-
-  @Benchmark
-  public void windowFromCurrentRow(Blackhole blackhole)
-  {
-    String sql = "SELECT COUNT(*) "
-                 + "OVER (PARTITION BY dimUniform ORDER BY dimSequential RANGE UNBOUNDED FOLLOWING) "
-                 + "FROM foo";
-    querySql(sql, blackhole);
-  }
-
-  @Benchmark
-  public void windowWithSorter(Blackhole blackhole)
-  {
-    String sql = "SELECT COUNT(*) "
-                 + "OVER (PARTITION BY dimUniform ORDER BY dimSequential) "
-                 + "FROM foo "
-                 + "GROUP BY dimSequential, dimUniform";
-    querySql(sql, blackhole);
-  }
-
-  @Benchmark
-  public void windowWithoutSorter(Blackhole blackhole)
-  {
-    String sql = "SELECT COUNT(*) "
-                 + "OVER (PARTITION BY dimUniform ORDER BY dimSequential) "
-                 + "FROM foo "
-                 + "GROUP BY dimUniform, dimSequential";
-    querySql(sql, blackhole);
-  }
-
-  @Benchmark
-  public void windowWithGroupbyTime(Blackhole blackhole)
-  {
-    String sql = "SELECT "
-                 + "SUM(dimSequentialHalfNull) + SUM(dimHyperUnique), "
-                 + "LAG(SUM(dimSequentialHalfNull + dimHyperUnique)) OVER (PARTITION BY dimUniform ORDER BY dimSequential) "
-                 + "FROM foo "
-                 + "GROUP BY __time, dimUniform, dimSequential";
-    querySql(sql, blackhole);
   }
 }
diff --git a/processing/src/main/java/org/apache/druid/segment/generator/GeneratorSchemaInfo.java b/processing/src/main/java/org/apache/druid/segment/generator/GeneratorSchemaInfo.java
index 603e7dd0d822..74aeea727cfb 100644
--- a/processing/src/main/java/org/apache/druid/segment/generator/GeneratorSchemaInfo.java
+++ b/processing/src/main/java/org/apache/druid/segment/generator/GeneratorSchemaInfo.java
@@ -26,7 +26,9 @@
 import org.apache.druid.timeline.partition.LinearShardSpec;
 import org.joda.time.Interval;
 
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 public class GeneratorSchemaInfo
@@ -64,6 +66,20 @@ public DimensionsSpec getDimensionsSpec()
     return new DimensionsSpec(specs);
   }
 
+  public DimensionsSpec getDimensionsSpecExcludeAggs()
+  {
+    final Set<String> metricsInputs = new HashSet<>();
+    for (AggregatorFactory agg : aggs) {
+      metricsInputs.addAll(agg.requiredFields());
+    }
+    List<DimensionSchema> specs = getColumnSchemas().stream()
+                                                    .filter(x -> !x.isMetric() && !metricsInputs.contains(x.getName()))
+                                                    .map(GeneratorColumnSchema::getDimensionSchema)
+                                                    .collect(Collectors.toList());
+
+    return new DimensionsSpec(specs);
+  }
+
   public List<AggregatorFactory> getAggs()
   {
     return aggs;
diff --git a/processing/src/test/java/org/apache/druid/segment/TestHelper.java b/processing/src/test/java/org/apache/druid/segment/TestHelper.java
index 8e0728730bee..ef9a91112163 100644
--- a/processing/src/test/java/org/apache/druid/segment/TestHelper.java
+++ b/processing/src/test/java/org/apache/druid/segment/TestHelper.java
@@ -67,6 +67,11 @@ public static IndexMergerV9 getTestIndexMergerV9(SegmentWriteOutMediumFactory se
     return new IndexMergerV9(JSON_MAPPER, getTestIndexIO(), segmentWriteOutMediumFactory, true);
   }
 
+  public static IndexMergerV9 getTestIndexMergerV9(ObjectMapper jsonMapper, SegmentWriteOutMediumFactory segmentWriteOutMediumFactory)
+  {
+    return new IndexMergerV9(jsonMapper, getTestIndexIO(jsonMapper), segmentWriteOutMediumFactory, true);
+  }
+
   public static IndexMergerV9 getTestIndexMergerV9(SegmentWriteOutMediumFactory segmentWriteOutMediumFactory, ColumnConfig columnConfig)
   {
     return new IndexMergerV9(JSON_MAPPER, getTestIndexIO(columnConfig), segmentWriteOutMediumFactory, true);
@@ -82,6 +87,16 @@ public static IndexIO getTestIndexIO(ColumnConfig columnConfig)
     return new IndexIO(JSON_MAPPER, columnConfig);
   }
 
+  public static IndexIO getTestIndexIO(ObjectMapper jsonMapper, ColumnConfig columnConfig)
+  {
+    return new IndexIO(jsonMapper, columnConfig);
+  }
+
+  public static IndexIO getTestIndexIO(ObjectMapper jsonMapper)
+  {
+    return new IndexIO(jsonMapper, ColumnConfig.SELECTION_SIZE);
+  }
+
   public static AnnotationIntrospector makeAnnotationIntrospector()
   {
     // Prepare annotationIntrospector with similar logic, except skip Guice loading
diff --git a/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java b/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java
index 5afb9fa04123..7f04463f73e3 100644
--- a/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java
+++ b/processing/src/test/java/org/apache/druid/segment/generator/SegmentGenerator.java
@@ -19,10 +19,12 @@
 
 package org.apache.druid.segment.generator;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.hash.Hashing;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.data.input.InputRow;
 import org.apache.druid.data.input.InputRowSchema;
+import org.apache.druid.data.input.impl.AggregateProjectionSpec;
 import org.apache.druid.data.input.impl.DimensionsSpec;
 import org.apache.druid.data.input.impl.MapInputRowParser;
 import org.apache.druid.data.input.impl.TimestampSpec;
@@ -55,6 +57,7 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -114,7 +117,16 @@ public QueryableIndex generate(
       final int numRows
   )
   {
-    return generate(dataSegment, schemaInfo, schemaInfo.getDimensionsSpec(), TransformSpec.NONE, IndexSpec.DEFAULT, granularity, numRows);
+    return generate(
+        dataSegment,
+        schemaInfo,
+        schemaInfo.getDimensionsSpec(),
+        TransformSpec.NONE,
+        IndexSpec.DEFAULT,
+        granularity,
+        Collections.emptyList(),
+        numRows
+    );
   }
 
   public QueryableIndex generate(
@@ -125,7 +137,16 @@ public QueryableIndex generate(
       final int numRows
   )
   {
-    return generate(dataSegment, schemaInfo, schemaInfo.getDimensionsSpec(), TransformSpec.NONE, indexSpec, granularity, numRows);
+    return generate(
+        dataSegment,
+        schemaInfo,
+        schemaInfo.getDimensionsSpec(),
+        TransformSpec.NONE,
+        indexSpec,
+        granularity,
+        Collections.emptyList(),
+        numRows
+    );
   }
 
   public QueryableIndex generate(
@@ -137,6 +158,53 @@ public QueryableIndex generate(
       final Granularity queryGranularity,
       final int numRows
   )
+  {
+    return generate(
+        dataSegment,
+        schemaInfo,
+        dimensionsSpec,
+        transformSpec,
+        indexSpec,
+        queryGranularity,
+        Collections.emptyList(),
+        numRows
+    );
+  }
+  public QueryableIndex generate(
+      final DataSegment dataSegment,
+      final GeneratorSchemaInfo schemaInfo,
+      final DimensionsSpec dimensionsSpec,
+      final TransformSpec transformSpec,
+      final IndexSpec indexSpec,
+      final Granularity queryGranularity,
+      final List<AggregateProjectionSpec> projectionSpecs,
+      final int numRows
+  )
+  {
+    return generate(
+        dataSegment,
+        schemaInfo,
+        dimensionsSpec,
+        transformSpec,
+        indexSpec,
+        queryGranularity,
+        projectionSpecs,
+        numRows,
+        TestHelper.JSON_MAPPER
+    );
+  }
+
+  public QueryableIndex generate(
+      final DataSegment dataSegment,
+      final GeneratorSchemaInfo schemaInfo,
+      final DimensionsSpec dimensionsSpec,
+      final TransformSpec transformSpec,
+      final IndexSpec indexSpec,
+      final Granularity queryGranularity,
+      final List<AggregateProjectionSpec> projectionSpecs,
+      final int numRows,
+      final ObjectMapper jsonMapper
+  )
   {
     // In case we need to generate hyperUniques or json
     ComplexMetrics.registerSerde(HyperUniquesSerde.TYPE_NAME, new HyperUniquesSerde());
@@ -149,6 +217,8 @@ public QueryableIndex generate(
                                    .putString(dimensionsSpec.toString(), StandardCharsets.UTF_8)
                                    .putString(queryGranularity.toString(), StandardCharsets.UTF_8)
                                    .putString(indexSpec.toString(), StandardCharsets.UTF_8)
+                                   .putString(transformSpec.toString(), StandardCharsets.UTF_8)
+                                   .putString(projectionSpecs.toString(), StandardCharsets.UTF_8)
                                    .putInt(numRows)
                                    .hash()
                                    .toString();
@@ -158,7 +228,7 @@ public QueryableIndex generate(
     if (outDir.exists()) {
       try {
         log.info("Found segment with hash[%s] cached in directory[%s].", dataHash, outDir);
-        return TestHelper.getTestIndexIO(ColumnConfig.DEFAULT).loadIndex(outDir);
+        return TestHelper.getTestIndexIO(jsonMapper, ColumnConfig.DEFAULT).loadIndex(outDir);
       }
       catch (IOException e) {
         throw new RuntimeException(e);
@@ -179,22 +249,23 @@ public QueryableIndex generate(
         .withMetrics(schemaInfo.getAggsArray())
         .withRollup(schemaInfo.isWithRollup())
         .withQueryGranularity(queryGranularity)
+        .withProjections(projectionSpecs)
         .build();
 
     final List<InputRow> rows = new ArrayList<>();
     final List<QueryableIndex> indexes = new ArrayList<>();
 
-    Transformer transformer = transformSpec.toTransformer();
-    InputRowSchema rowSchema = new InputRowSchema(
+    final Transformer transformer = transformSpec.toTransformer();
+    final InputRowSchema rowSchema = new InputRowSchema(
         new TimestampSpec(null, null, null),
         dimensionsSpec,
         null
     );
 
     for (int i = 0; i < numRows; i++) {
-      Map<String, Object> raw = dataGenerator.nextRaw();
-      InputRow inputRow = MapInputRowParser.parse(rowSchema, raw);
-      InputRow transformedRow = transformer.transform(inputRow);
+      final Map<String, Object> raw = dataGenerator.nextRaw();
+      final InputRow inputRow = MapInputRowParser.parse(rowSchema, raw);
+      final InputRow transformedRow = transformer.transform(inputRow);
       rows.add(transformedRow);
 
       if ((i + 1) % 20000 == 0) {
@@ -202,7 +273,7 @@ public QueryableIndex generate(
       }
 
       if (rows.size() % MAX_ROWS_IN_MEMORY == 0) {
-        indexes.add(makeIndex(dataSegment.getId(), dataHash, indexes.size(), rows, indexSchema, indexSpec));
+        indexes.add(makeIndex(dataSegment.getId(), dataHash, indexes.size(), rows, indexSchema, indexSpec, jsonMapper));
         rows.clear();
       }
     }
@@ -210,7 +281,7 @@ public QueryableIndex generate(
     log.info("%,d/%,d rows generated for[%s].", numRows, numRows, dataSegment);
 
     if (rows.size() > 0) {
-      indexes.add(makeIndex(dataSegment.getId(), dataHash, indexes.size(), rows, indexSchema, indexSpec));
+      indexes.add(makeIndex(dataSegment.getId(), dataHash, indexes.size(), rows, indexSchema, indexSpec, jsonMapper));
       rows.clear();
     }
 
@@ -221,9 +292,9 @@ public QueryableIndex generate(
     } else {
       try {
         retVal = TestHelper
-            .getTestIndexIO(ColumnConfig.DEFAULT)
+            .getTestIndexIO(jsonMapper, ColumnConfig.DEFAULT)
             .loadIndex(
-                TestHelper.getTestIndexMergerV9(OffHeapMemorySegmentWriteOutMediumFactory.instance())
+                TestHelper.getTestIndexMergerV9(jsonMapper, OffHeapMemorySegmentWriteOutMediumFactory.instance())
                           .mergeQueryableIndex(
                               indexes,
                               false,
@@ -256,7 +327,82 @@ public QueryableIndex generate(
   }
 
   public IncrementalIndex generateIncrementalIndex(
+      final DataSegment dataSegment,
+      final GeneratorSchemaInfo schemaInfo,
+      final DimensionsSpec dimensionsSpec,
+      final TransformSpec transformSpec,
+      final AggregatorFactory[] aggregatorFactories,
+      final IndexSpec indexSpec,
+      final Granularity queryGranularity,
+      final List<AggregateProjectionSpec> projectionSpecs,
+      final int numRows,
+      final ObjectMapper jsonMapper
+  )
+  {
+    // In case we need to generate hyperUniques or json
+    ComplexMetrics.registerSerde(HyperUniquesSerde.TYPE_NAME, new HyperUniquesSerde());
+    BuiltInTypesModule.registerHandlersAndSerde();
+
+    final String dataHash = Hashing.sha256()
+                                   .newHasher()
+                                   .putString(dataSegment.getId().toString(), StandardCharsets.UTF_8)
+                                   .putString(schemaInfo.toString(), StandardCharsets.UTF_8)
+                                   .putString(dimensionsSpec.toString(), StandardCharsets.UTF_8)
+                                   .putString(queryGranularity.toString(), StandardCharsets.UTF_8)
+                                   .putString(indexSpec.toString(), StandardCharsets.UTF_8)
+                                   .putInt(numRows)
+                                   .hash()
+                                   .toString();
+
+    log.info("Writing segment with hash[%s] to incremental index.", dataHash);
+
+    final DataGenerator dataGenerator = new DataGenerator(
+        schemaInfo.getColumnSchemas(),
+        dataSegment.getId().hashCode(), /* Use segment identifier hashCode as seed */
+        schemaInfo.getDataInterval(),
+        numRows
+    );
+
+    final IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder()
+        .withDimensionsSpec(dimensionsSpec)
+        .withMetrics(aggregatorFactories)
+        .withRollup(schemaInfo.isWithRollup())
+        .withQueryGranularity(queryGranularity)
+        .withProjections(projectionSpecs)
+        .build();
+
+    final List<InputRow> rows = new ArrayList<>();
 
+    final Transformer transformer = transformSpec.toTransformer();
+    final InputRowSchema rowSchema = new InputRowSchema(
+        new TimestampSpec(null, null, null),
+        dimensionsSpec,
+        null
+    );
+
+    for (int i = 0; i < numRows; i++) {
+      final Map<String, Object> raw = dataGenerator.nextRaw();
+      final InputRow inputRow = MapInputRowParser.parse(rowSchema, raw);
+      final InputRow transformedRow = transformer.transform(inputRow);
+      rows.add(transformedRow);
+
+      if ((i + 1) % 20000 == 0) {
+        log.info("%,d/%,d rows generated for[%s].", i + 1, numRows, dataSegment);
+      }
+    }
+
+    log.info("%,d/%,d rows generated for[%s].", numRows, numRows, dataSegment);
+
+    return IndexBuilder
+        .create(jsonMapper)
+        .schema(indexSchema)
+        .tmpDir(new File(getSegmentDir(dataSegment.getId(), dataHash), ""))
+        .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
+        .rows(rows)
+        .buildIncrementalIndex();
+  }
+
+  public IncrementalIndex generateIncrementalIndex(
       final DataSegment dataSegment,
       final GeneratorSchemaInfo schemaInfo,
       final Granularity granularity,
@@ -320,11 +466,12 @@ private QueryableIndex makeIndex(
       final int indexNumber,
       final List<InputRow> rows,
       final IncrementalIndexSchema indexSchema,
-      final IndexSpec indexSpec
+      final IndexSpec indexSpec,
+      final ObjectMapper jsonMapper
   )
   {
     return IndexBuilder
-        .create()
+        .create(jsonMapper)
         .schema(indexSchema)
         .indexSpec(indexSpec)
         .tmpDir(new File(getSegmentDir(identifier, dataHash), String.valueOf(indexNumber)))

From 60daddedf8b0a9faf1edeb4603eb4b8e86fd3dc2 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Tue, 22 Oct 2024 20:21:21 -0700
Subject: [PATCH 07/12] SeekableStreamSupervisor: Use workerExec as the client
 connectExec. (#17394)

* SeekableStreamSupervisor: Use workerExec as the client connectExec.

This patch uses the already-existing per-supervisor workerExec as the
connectExec for task clients, rather than using the process-wide default
ServiceClientFactory pool.

This helps prevent callbacks from backlogging on the process-wide pool.
It's especially useful for retries, where callbacks may need to establish
new TCP connections or perform TLS handshakes.

* Fix compilation, tests.

* Fix style.
---
 .../RabbitStreamIndexTaskClientFactory.java   |   6 +-
 .../kafka/KafkaIndexTaskClientFactory.java    |   6 +-
 .../kafka/supervisor/KafkaSupervisorTest.java | 999 +++++++++++++-----
 .../KinesisIndexTaskClientFactory.java        |   6 +-
 .../supervisor/KinesisSupervisorTest.java     |  24 +-
 .../SeekableStreamIndexTaskClientFactory.java |  27 +-
 .../supervisor/SeekableStreamSupervisor.java  |  36 +-
 .../SeekableStreamSupervisorStateTest.java    |   2 +-
 8 files changed, 785 insertions(+), 321 deletions(-)

diff --git a/extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskClientFactory.java b/extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskClientFactory.java
index 83cdde8191ff..088dc6b77d53 100644
--- a/extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskClientFactory.java
+++ b/extensions-contrib/rabbit-stream-indexing-service/src/main/java/org/apache/druid/indexing/rabbitstream/RabbitStreamIndexTaskClientFactory.java
@@ -25,17 +25,17 @@
 import org.apache.druid.guice.annotations.EscalatedGlobal;
 import org.apache.druid.guice.annotations.Json;
 import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory;
-import org.apache.druid.rpc.ServiceClientFactory;
+import org.apache.druid.java.util.http.client.HttpClient;
 
 @LazySingleton
 public class RabbitStreamIndexTaskClientFactory extends SeekableStreamIndexTaskClientFactory<String, Long>
 {
   @Inject
   public RabbitStreamIndexTaskClientFactory(
-      @EscalatedGlobal ServiceClientFactory serviceClientFactory,
+      @EscalatedGlobal HttpClient httpClient,
       @Json ObjectMapper mapper)
   {
-    super(serviceClientFactory, mapper);
+    super(httpClient, mapper);
   }
 
   @Override
diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTaskClientFactory.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTaskClientFactory.java
index 73a880b524b7..10b57dd4757f 100644
--- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTaskClientFactory.java
+++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/KafkaIndexTaskClientFactory.java
@@ -26,18 +26,18 @@
 import org.apache.druid.guice.annotations.EscalatedGlobal;
 import org.apache.druid.guice.annotations.Json;
 import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory;
-import org.apache.druid.rpc.ServiceClientFactory;
+import org.apache.druid.java.util.http.client.HttpClient;
 
 @LazySingleton
 public class KafkaIndexTaskClientFactory extends SeekableStreamIndexTaskClientFactory<KafkaTopicPartition, Long>
 {
   @Inject
   public KafkaIndexTaskClientFactory(
-      @EscalatedGlobal ServiceClientFactory serviceClientFactory,
+      @EscalatedGlobal HttpClient httpClient,
       @Json ObjectMapper mapper
   )
   {
-    super(serviceClientFactory, mapper);
+    super(httpClient, mapper);
   }
 
   @Override
diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
index e436b8cd56a5..7926a0568fd9 100644
--- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
+++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java
@@ -127,6 +127,7 @@
 import java.util.Properties;
 import java.util.TreeMap;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 
 @RunWith(Parameterized.class)
 public class KafkaSupervisorTest extends EasyMockSupport
@@ -202,10 +203,11 @@ public static void setupClass() throws Exception
         zkServer.getConnectString(),
         null,
         1,
-        ImmutableMap.of("num.partitions",
-                        String.valueOf(NUM_PARTITIONS),
-                        "auto.create.topics.enable",
-                        String.valueOf(false)
+        ImmutableMap.of(
+            "num.partitions",
+            String.valueOf(NUM_PARTITIONS),
+            "auto.create.topics.enable",
+            String.valueOf(false)
         )
     );
     kafkaServer.start();
@@ -267,11 +269,10 @@ public void testNoInitialStateWithAutoscaler() throws Exception
       public SeekableStreamIndexTaskClient<KafkaTopicPartition, Long> build(
           final String dataSource,
           final TaskInfoProvider taskInfoProvider,
-          final int maxNumTasks,
-          final SeekableStreamSupervisorTuningConfig tuningConfig
+          final SeekableStreamSupervisorTuningConfig tuningConfig,
+          final ScheduledExecutorService connectExec
       )
       {
-        Assert.assertEquals(replicas * taskCountMax, maxNumTasks);
         Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), tuningConfig.getHttpTimeout());
         Assert.assertEquals(TEST_CHAT_RETRIES, (long) tuningConfig.getChatRetries());
         return taskClient;
@@ -299,53 +300,53 @@ public SeekableStreamIndexTaskClient<KafkaTopicPartition, Long> build(
     consumerProperties.put("bootstrap.servers", kafkaHost);
 
     KafkaSupervisorIOConfig kafkaSupervisorIOConfig = new KafkaSupervisorIOConfig(
-            topic,
-            null,
-            INPUT_FORMAT,
-            replicas,
-            1,
-            new Period("PT1H"),
-            consumerProperties,
-            OBJECT_MAPPER.convertValue(autoScalerConfig, LagBasedAutoScalerConfig.class),
-            KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS,
-            new Period("P1D"),
-            new Period("PT30S"),
-            true,
-            new Period("PT30M"),
-            null,
-            null,
-            null,
-            null,
-            new IdleConfig(true, 1000L),
-            1
+        topic,
+        null,
+        INPUT_FORMAT,
+        replicas,
+        1,
+        new Period("PT1H"),
+        consumerProperties,
+        OBJECT_MAPPER.convertValue(autoScalerConfig, LagBasedAutoScalerConfig.class),
+        KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS,
+        new Period("P1D"),
+        new Period("PT30S"),
+        true,
+        new Period("PT30M"),
+        null,
+        null,
+        null,
+        null,
+        new IdleConfig(true, 1000L),
+        1
     );
 
     final KafkaSupervisorTuningConfig tuningConfigOri = new KafkaSupervisorTuningConfig(
-            null,
-            1000,
-            null,
-            null,
-            50000,
-            null,
-            new Period("P1Y"),
-            null,
-            null,
-            null,
-            false,
-            null,
-            false,
-            null,
-            numThreads,
-            TEST_CHAT_RETRIES,
-            TEST_HTTP_TIMEOUT,
-            TEST_SHUTDOWN_TIMEOUT,
-            null,
-            null,
-            null,
-            null,
-            null,
-            null,
-          null
+        null,
+        1000,
+        null,
+        null,
+        50000,
+        null,
+        new Period("P1Y"),
+        null,
+        null,
+        null,
+        false,
+        null,
+        false,
+        null,
+        numThreads,
+        TEST_CHAT_RETRIES,
+        TEST_HTTP_TIMEOUT,
+        TEST_SHUTDOWN_TIMEOUT,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null,
+        null
     );
 
     EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(kafkaSupervisorIOConfig).anyTimes();
@@ -354,31 +355,31 @@ public SeekableStreamIndexTaskClient<KafkaTopicPartition, Long> build(
     EasyMock.replay(ingestionSchema);
 
     SeekableStreamSupervisorSpec testableSupervisorSpec = new KafkaSupervisorSpec(
-            ingestionSchema,
-            dataSchema,
-            tuningConfigOri,
-            kafkaSupervisorIOConfig,
-            null,
-            false,
-            taskStorage,
-            taskMaster,
-            indexerMetadataStorageCoordinator,
-            taskClientFactory,
-            OBJECT_MAPPER,
-            new NoopServiceEmitter(),
-            new DruidMonitorSchedulerConfig(),
-            rowIngestionMetersFactory,
-            new SupervisorStateManagerConfig()
+        ingestionSchema,
+        dataSchema,
+        tuningConfigOri,
+        kafkaSupervisorIOConfig,
+        null,
+        false,
+        taskStorage,
+        taskMaster,
+        indexerMetadataStorageCoordinator,
+        taskClientFactory,
+        OBJECT_MAPPER,
+        new NoopServiceEmitter(),
+        new DruidMonitorSchedulerConfig(),
+        rowIngestionMetersFactory,
+        new SupervisorStateManagerConfig()
     );
 
     supervisor = new TestableKafkaSupervisor(
-            taskStorage,
-            taskMaster,
-            indexerMetadataStorageCoordinator,
-            taskClientFactory,
-            OBJECT_MAPPER,
-            (KafkaSupervisorSpec) testableSupervisorSpec,
-            rowIngestionMetersFactory
+        taskStorage,
+        taskMaster,
+        indexerMetadataStorageCoordinator,
+        taskClientFactory,
+        OBJECT_MAPPER,
+        (KafkaSupervisorSpec) testableSupervisorSpec,
+        rowIngestionMetersFactory
     );
 
     SupervisorTaskAutoScaler autoscaler = testableSupervisorSpec.createAutoscaler(supervisor);
@@ -393,9 +394,9 @@ public SeekableStreamIndexTaskClient<KafkaTopicPartition, Long> build(
     EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes();
     EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
     EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(
-            new KafkaDataSourceMetadata(
-                    null
-            )
+        new KafkaDataSourceMetadata(
+            null
+        )
     ).anyTimes();
     EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true).anyTimes();
     taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
@@ -425,22 +426,43 @@ public SeekableStreamIndexTaskClient<KafkaTopicPartition, Long> build(
     Assert.assertFalse("maximumMessageTime", taskConfig.getMaximumMessageTime().isPresent());
 
     Assert.assertEquals(topic, taskConfig.getStartSequenceNumbers().getStream());
-    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)));
-    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)));
-    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)));
+    Assert.assertEquals(
+        0L,
+        (long) taskConfig.getStartSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 0))
+    );
+    Assert.assertEquals(
+        0L,
+        (long) taskConfig.getStartSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 1))
+    );
+    Assert.assertEquals(
+        0L,
+        (long) taskConfig.getStartSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 2))
+    );
 
     Assert.assertEquals(topic, taskConfig.getEndSequenceNumbers().getStream());
     Assert.assertEquals(
-            Long.MAX_VALUE,
-            (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0))
+        Long.MAX_VALUE,
+        (long) taskConfig.getEndSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 0))
     );
     Assert.assertEquals(
-            Long.MAX_VALUE,
-            (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1))
+        Long.MAX_VALUE,
+        (long) taskConfig.getEndSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 1))
     );
     Assert.assertEquals(
-            Long.MAX_VALUE,
-            (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2))
+        Long.MAX_VALUE,
+        (long) taskConfig.getEndSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 2))
     );
     Assert.assertEquals(
         Collections.singleton(new ResourceAction(
@@ -543,22 +565,43 @@ public void testNoInitialState() throws Exception
     Assert.assertFalse("maximumMessageTime", taskConfig.getMaximumMessageTime().isPresent());
 
     Assert.assertEquals(topic, taskConfig.getStartSequenceNumbers().getStream());
-    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)));
-    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)));
-    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)));
+    Assert.assertEquals(
+        0L,
+        (long) taskConfig.getStartSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 0))
+    );
+    Assert.assertEquals(
+        0L,
+        (long) taskConfig.getStartSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 1))
+    );
+    Assert.assertEquals(
+        0L,
+        (long) taskConfig.getStartSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 2))
+    );
 
     Assert.assertEquals(topic, taskConfig.getEndSequenceNumbers().getStream());
     Assert.assertEquals(
         Long.MAX_VALUE,
-        (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0))
+        (long) taskConfig.getEndSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 0))
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1))
+        (long) taskConfig.getEndSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 1))
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2))
+        (long) taskConfig.getEndSequenceNumbers()
+                         .getPartitionSequenceNumberMap()
+                         .get(new KafkaTopicPartition(false, topic, 2))
     );
   }
 
@@ -613,19 +656,35 @@ public void testMultiTask() throws Exception
     Assert.assertEquals(2, task1.getIOConfig().getEndSequenceNumbers().getPartitionSequenceNumberMap().size());
     Assert.assertEquals(
         0L,
-        task1.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        task1.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 0))
+             .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        task1.getIOConfig().getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        task1.getIOConfig()
+             .getEndSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 0))
+             .longValue()
     );
     Assert.assertEquals(
         0L,
-        task1.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        task1.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 2))
+             .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        task1.getIOConfig().getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        task1.getIOConfig()
+             .getEndSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 2))
+             .longValue()
     );
 
     KafkaIndexTask task2 = captured.getValues().get(1);
@@ -633,11 +692,19 @@ public void testMultiTask() throws Exception
     Assert.assertEquals(1, task2.getIOConfig().getEndSequenceNumbers().getPartitionSequenceNumberMap().size());
     Assert.assertEquals(
         0L,
-        task2.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        task2.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 1))
+             .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        task2.getIOConfig().getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        task2.getIOConfig()
+             .getEndSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 1))
+             .longValue()
     );
   }
 
@@ -668,15 +735,27 @@ public void testReplicas() throws Exception
     Assert.assertEquals(3, task1.getIOConfig().getEndSequenceNumbers().getPartitionSequenceNumberMap().size());
     Assert.assertEquals(
         0L,
-        task1.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        task1.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 0))
+             .longValue()
     );
     Assert.assertEquals(
         0L,
-        task1.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        task1.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 1))
+             .longValue()
     );
     Assert.assertEquals(
         0L,
-        task1.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        task1.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 2))
+             .longValue()
     );
 
     KafkaIndexTask task2 = captured.getValues().get(1);
@@ -684,15 +763,27 @@ public void testReplicas() throws Exception
     Assert.assertEquals(3, task2.getIOConfig().getEndSequenceNumbers().getPartitionSequenceNumberMap().size());
     Assert.assertEquals(
         0L,
-        task2.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        task2.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 0))
+             .longValue()
     );
     Assert.assertEquals(
         0L,
-        task2.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        task2.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 1))
+             .longValue()
     );
     Assert.assertEquals(
         0L,
-        task2.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        task2.getIOConfig()
+             .getStartSequenceNumbers()
+             .getPartitionSequenceNumberMap()
+             .get(new KafkaTopicPartition(false, topic, 2))
+             .longValue()
     );
   }
 
@@ -802,15 +893,27 @@ public void testLatestOffset() throws Exception
     KafkaIndexTask task = captured.getValue();
     Assert.assertEquals(
         1101L,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 0))
+            .longValue()
     );
     Assert.assertEquals(
         1101L,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 1))
+            .longValue()
     );
     Assert.assertEquals(
         1101L,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 2))
+            .longValue()
     );
   }
 
@@ -867,15 +970,27 @@ public void testAlwaysUsesEarliestOffsetForNewlyDiscoveredPartitions() throws Ex
     KafkaIndexTask task = captured.getValue();
     Assert.assertEquals(
         10,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 0))
+            .longValue()
     );
     Assert.assertEquals(
         10,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 1))
+            .longValue()
     );
     Assert.assertEquals(
         10,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 2))
+            .longValue()
     );
 
     addMoreEvents(9, 6);
@@ -899,15 +1014,27 @@ public void testAlwaysUsesEarliestOffsetForNewlyDiscoveredPartitions() throws Ex
     task = newcaptured.getValue();
     Assert.assertEquals(
         0,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 3)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 3))
+            .longValue()
     );
     Assert.assertEquals(
         0,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 4)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 4))
+            .longValue()
     );
     Assert.assertEquals(
         0,
-        task.getIOConfig().getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 5)).longValue()
+        task.getIOConfig()
+            .getStartSequenceNumbers()
+            .getPartitionSequenceNumberMap()
+            .get(new KafkaTopicPartition(false, topic, 5))
+            .longValue()
     );
   }
 
@@ -928,7 +1055,11 @@ public void testDatasourceMetadataSingleTopicToSingleTopicMatch() throws Excepti
     EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
     EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(
         new KafkaDataSourceMetadata(
-            new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of())
+            new SeekableStreamStartSequenceNumbers<>(
+                topic,
+                singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+                ImmutableSet.of()
+            )
         )
     ).anyTimes();
     EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true);
@@ -943,15 +1074,24 @@ public void testDatasourceMetadataSingleTopicToSingleTopicMatch() throws Excepti
     Assert.assertEquals("sequenceName-0", taskConfig.getBaseSequenceName());
     Assert.assertEquals(
         10L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 0))
+                  .longValue()
     );
     Assert.assertEquals(
         20L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 1))
+                  .longValue()
     );
     Assert.assertEquals(
         30L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 2))
+                  .longValue()
     );
   }
 
@@ -991,15 +1131,24 @@ public void testDatasourceMetadataSingleTopicToMultiTopicMatch() throws Exceptio
     Assert.assertEquals("sequenceName-0", taskConfig.getBaseSequenceName());
     Assert.assertEquals(
         10L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 0)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 0))
+                  .longValue()
     );
     Assert.assertEquals(
         20L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 1)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 1))
+                  .longValue()
     );
     Assert.assertEquals(
         30L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 2)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 2))
+                  .longValue()
     );
     Assert.assertEquals(3, taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().size());
   }
@@ -1040,15 +1189,24 @@ public void testDatasourceMetadataSingleTopicToMultiTopicNotMatch() throws Excep
     Assert.assertEquals("sequenceName-0", taskConfig.getBaseSequenceName());
     Assert.assertEquals(
         0L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 0)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 0))
+                  .longValue()
     );
     Assert.assertEquals(
         0L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 1)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 1))
+                  .longValue()
     );
     Assert.assertEquals(
         0L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 2)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 2))
+                  .longValue()
     );
     Assert.assertEquals(3, taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().size());
   }
@@ -1074,7 +1232,11 @@ public void testDatasourceMetadataMultiTopicToSingleTopic() throws Exception
     partitionSequenceNumberMap.putAll(multiTopicPartitionMap("notMatch", 0, 10L, 1, 20L, 2, 30L));
     EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(
         new KafkaDataSourceMetadata(
-            new SeekableStreamStartSequenceNumbers<>(topicPattern, partitionSequenceNumberMap.build(), ImmutableSet.of())
+            new SeekableStreamStartSequenceNumbers<>(
+                topicPattern,
+                partitionSequenceNumberMap.build(),
+                ImmutableSet.of()
+            )
         )
     ).anyTimes();
     EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true);
@@ -1089,15 +1251,24 @@ public void testDatasourceMetadataMultiTopicToSingleTopic() throws Exception
     Assert.assertEquals("sequenceName-0", taskConfig.getBaseSequenceName());
     Assert.assertEquals(
         10L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 0))
+                  .longValue()
     );
     Assert.assertEquals(
         20L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 1))
+                  .longValue()
     );
     Assert.assertEquals(
         30L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 2))
+                  .longValue()
     );
     Assert.assertEquals(3, taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().size());
   }
@@ -1139,15 +1310,24 @@ public void testDatasourceMetadataMultiTopicToMultiTopic() throws Exception
     Assert.assertEquals("sequenceName-0", taskConfig.getBaseSequenceName());
     Assert.assertEquals(
         10L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 0)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 0))
+                  .longValue()
     );
     Assert.assertEquals(
         20L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 1)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 1))
+                  .longValue()
     );
     Assert.assertEquals(
         30L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(true, topic, 2)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(true, topic, 2))
+                  .longValue()
     );
     Assert.assertEquals(3, taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().size());
   }
@@ -1239,7 +1419,10 @@ public void testKillBadPartitionAssignment() throws Exception
         DATASOURCE,
         0,
         new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 2, 0L), ImmutableSet.of()),
-        new SeekableStreamEndSequenceNumbers<>("topic", singlePartitionMap(topic, 0, Long.MAX_VALUE, 2, Long.MAX_VALUE)),
+        new SeekableStreamEndSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, Long.MAX_VALUE, 2, Long.MAX_VALUE)
+        ),
         null,
         null,
         tuningConfig
@@ -1258,7 +1441,11 @@ public void testKillBadPartitionAssignment() throws Exception
         "id3",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -1272,7 +1459,10 @@ public void testKillBadPartitionAssignment() throws Exception
         DATASOURCE,
         0,
         new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 1, 0L), ImmutableSet.of()),
-        new SeekableStreamEndSequenceNumbers<>("topic", singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE)),
+        new SeekableStreamEndSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE)
+        ),
         null,
         null,
         tuningConfig
@@ -1282,7 +1472,10 @@ public void testKillBadPartitionAssignment() throws Exception
         DATASOURCE,
         0,
         new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 1, 0L, 2, 0L), ImmutableSet.of()),
-        new SeekableStreamEndSequenceNumbers<>("topic", singlePartitionMap(topic, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)),
+        new SeekableStreamEndSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
+        ),
         null,
         null,
         tuningConfig
@@ -1443,7 +1636,10 @@ public void testRequeueAdoptedTaskWhenFailed() throws Exception
         DATASOURCE,
         0,
         new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 2, 0L), ImmutableSet.of()),
-        new SeekableStreamEndSequenceNumbers<>("topic", singlePartitionMap(topic, 0, Long.MAX_VALUE, 2, Long.MAX_VALUE)),
+        new SeekableStreamEndSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, Long.MAX_VALUE, 2, Long.MAX_VALUE)
+        ),
         now,
         maxi,
         supervisor.getTuningConfig()
@@ -1726,8 +1922,18 @@ public void testBeginPublishAndQueueNextTasks() throws Exception
       Assert.assertTrue("isUseTransaction", taskConfig.isUseTransaction());
 
       Assert.assertEquals(topic, taskConfig.getStartSequenceNumbers().getStream());
-      Assert.assertEquals(10L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)));
-      Assert.assertEquals(35L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)));
+      Assert.assertEquals(
+          10L,
+          (long) taskConfig.getStartSequenceNumbers()
+                           .getPartitionSequenceNumberMap()
+                           .get(new KafkaTopicPartition(false, topic, 0))
+      );
+      Assert.assertEquals(
+          35L,
+          (long) taskConfig.getStartSequenceNumbers()
+                           .getPartitionSequenceNumberMap()
+                           .get(new KafkaTopicPartition(false, topic, 2))
+      );
     }
   }
 
@@ -1744,7 +1950,11 @@ public void testDiscoverExistingPublishingTask() throws Exception
         "id1",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -1825,29 +2035,47 @@ public void testDiscoverExistingPublishingTask() throws Exception
     Assert.assertEquals(topic, capturedTaskConfig.getStartSequenceNumbers().getStream());
     Assert.assertEquals(
         10L,
-        capturedTaskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        capturedTaskConfig.getStartSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 0))
+                          .longValue()
     );
     Assert.assertEquals(
         20L,
-        capturedTaskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        capturedTaskConfig.getStartSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 1))
+                          .longValue()
     );
     Assert.assertEquals(
         30L,
-        capturedTaskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        capturedTaskConfig.getStartSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 2))
+                          .longValue()
     );
 
     Assert.assertEquals(topic, capturedTaskConfig.getEndSequenceNumbers().getStream());
     Assert.assertEquals(
         Long.MAX_VALUE,
-        capturedTaskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        capturedTaskConfig.getEndSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 0))
+                          .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        capturedTaskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        capturedTaskConfig.getEndSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 1))
+                          .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        capturedTaskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        capturedTaskConfig.getEndSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 2))
+                          .longValue()
     );
   }
 
@@ -1865,7 +2093,10 @@ public void testDiscoverExistingPublishingTaskWithDifferentPartitionAllocation()
         DATASOURCE,
         0,
         new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 2, 0L), ImmutableSet.of()),
-        new SeekableStreamEndSequenceNumbers<>("topic", singlePartitionMap(topic, 0, Long.MAX_VALUE, 2, Long.MAX_VALUE)),
+        new SeekableStreamEndSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, Long.MAX_VALUE, 2, Long.MAX_VALUE)
+        ),
         null,
         null,
         supervisor.getTuningConfig()
@@ -1936,29 +2167,47 @@ public void testDiscoverExistingPublishingTaskWithDifferentPartitionAllocation()
     Assert.assertEquals(topic, capturedTaskConfig.getStartSequenceNumbers().getStream());
     Assert.assertEquals(
         10L,
-        capturedTaskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        capturedTaskConfig.getStartSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 0))
+                          .longValue()
     );
     Assert.assertEquals(
         0L,
-        capturedTaskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        capturedTaskConfig.getStartSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 1))
+                          .longValue()
     );
     Assert.assertEquals(
         30L,
-        capturedTaskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        capturedTaskConfig.getStartSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 2))
+                          .longValue()
     );
 
     Assert.assertEquals(topic, capturedTaskConfig.getEndSequenceNumbers().getStream());
     Assert.assertEquals(
         Long.MAX_VALUE,
-        capturedTaskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        capturedTaskConfig.getEndSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 0))
+                          .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        capturedTaskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        capturedTaskConfig.getEndSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 1))
+                          .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        capturedTaskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        capturedTaskConfig.getEndSequenceNumbers()
+                          .getPartitionSequenceNumberMap()
+                          .get(new KafkaTopicPartition(false, topic, 2))
+                          .longValue()
     );
   }
 
@@ -1977,7 +2226,11 @@ public void testDiscoverExistingPublishingAndReadingTask() throws Exception
         "id1",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -1991,7 +2244,11 @@ public void testDiscoverExistingPublishingAndReadingTask() throws Exception
         "id2",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 1L, 1, 2L, 2, 3L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 1L, 1, 2L, 2, 3L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -2407,7 +2664,17 @@ public void testSupervisorIsIdleIfStreamInactiveWhenNoActiveTasks() throws Excep
     taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
     EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(
         new KafkaDataSourceMetadata(
-            new SeekableStreamEndSequenceNumbers<KafkaTopicPartition, Long>(topic, singlePartitionMap(topic, 0, 2L, 1, 2L, 2, 2L))
+            new SeekableStreamEndSequenceNumbers<KafkaTopicPartition, Long>(
+                topic,
+                singlePartitionMap(topic,
+                                   0,
+                                   2L,
+                                   1,
+                                   2L,
+                                   2,
+                                   2L
+                )
+            )
         )
     ).anyTimes();
     EasyMock.expect(taskQueue.add(EasyMock.anyObject())).andReturn(true).anyTimes();
@@ -2455,7 +2722,17 @@ public void testSupervisorNotIdleIfStreamInactiveWhenSuspended() throws Exceptio
     taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
     EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(
         new KafkaDataSourceMetadata(
-            new SeekableStreamEndSequenceNumbers<KafkaTopicPartition, Long>(topic, singlePartitionMap(topic, 0, 2L, 1, 2L, 2, 2L))
+            new SeekableStreamEndSequenceNumbers<KafkaTopicPartition, Long>(
+                topic,
+                singlePartitionMap(topic,
+                                   0,
+                                   2L,
+                                   1,
+                                   2L,
+                                   2,
+                                   2L
+                )
+            )
         )
     ).anyTimes();
 
@@ -2484,7 +2761,8 @@ public void testSupervisorNotIdleIfStreamInactiveWhenSuspended() throws Exceptio
   public void testSupervisorIsIdleIfStreamInactiveWhenSuspended() throws Exception
   {
     Map<String, String> config = ImmutableMap.of("idleConfig.enabled", "false",
-                                                 "idleConfig.inactiveAfterMillis", "200");
+                                                 "idleConfig.inactiveAfterMillis", "200"
+    );
     supervisorConfig = OBJECT_MAPPER.convertValue(config, SupervisorStateManagerConfig.class);
     supervisor = getTestableSupervisorForIdleBehaviour(
         1,
@@ -2504,7 +2782,17 @@ public void testSupervisorIsIdleIfStreamInactiveWhenSuspended() throws Exception
     taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
     EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(
         new KafkaDataSourceMetadata(
-            new SeekableStreamEndSequenceNumbers<KafkaTopicPartition, Long>(topic, singlePartitionMap(topic, 0, 2L, 1, 2L, 2, 2L))
+            new SeekableStreamEndSequenceNumbers<KafkaTopicPartition, Long>(
+                topic,
+                singlePartitionMap(topic,
+                                   0,
+                                   2L,
+                                   1,
+                                   2L,
+                                   2,
+                                   2L
+                )
+            )
         )
     ).anyTimes();
     EasyMock.expect(taskQueue.add(EasyMock.anyObject())).andReturn(true).anyTimes();
@@ -2836,8 +3124,18 @@ public void testKillUnresponsiveTasksWhilePausing() throws Exception
 
     for (Task task : captured.getValues()) {
       KafkaIndexTaskIOConfig taskConfig = ((KafkaIndexTask) task).getIOConfig();
-      Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)));
-      Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)));
+      Assert.assertEquals(
+          0L,
+          (long) taskConfig.getStartSequenceNumbers()
+                           .getPartitionSequenceNumberMap()
+                           .get(new KafkaTopicPartition(false, topic, 0))
+      );
+      Assert.assertEquals(
+          0L,
+          (long) taskConfig.getStartSequenceNumbers()
+                           .getPartitionSequenceNumberMap()
+                           .get(new KafkaTopicPartition(false, topic, 2))
+      );
     }
   }
 
@@ -2930,8 +3228,18 @@ public void testKillUnresponsiveTasksWhileSettingEndOffsets() throws Exception
 
     for (Task task : captured.getValues()) {
       KafkaIndexTaskIOConfig taskConfig = ((KafkaIndexTask) task).getIOConfig();
-      Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)));
-      Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)));
+      Assert.assertEquals(
+          0L,
+          (long) taskConfig.getStartSequenceNumbers()
+                           .getPartitionSequenceNumberMap()
+                           .get(new KafkaTopicPartition(false, topic, 0))
+      );
+      Assert.assertEquals(
+          0L,
+          (long) taskConfig.getStartSequenceNumbers()
+                           .getPartitionSequenceNumberMap()
+                           .get(new KafkaTopicPartition(false, topic, 2))
+      );
     }
   }
 
@@ -2972,7 +3280,11 @@ public void testStopGracefully() throws Exception
         "id1",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -2986,7 +3298,11 @@ public void testStopGracefully() throws Exception
         "id2",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3000,7 +3316,11 @@ public void testStopGracefully() throws Exception
         "id3",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3132,7 +3452,11 @@ public void testResetDataSourceMetadata() throws Exception
     );
 
     KafkaDataSourceMetadata resetMetadata = new KafkaDataSourceMetadata(
-        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 1, 1000L, 2, 1000L), ImmutableSet.of())
+        new SeekableStreamStartSequenceNumbers<>(
+            topic,
+            singlePartitionMap(topic, 1, 1000L, 2, 1000L),
+            ImmutableSet.of()
+        )
     );
 
     KafkaDataSourceMetadata expectedMetadata = new KafkaDataSourceMetadata(
@@ -3249,7 +3573,11 @@ public void testResetRunningTasks() throws Exception
         "id1",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3263,7 +3591,11 @@ public void testResetRunningTasks() throws Exception
         "id2",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3277,7 +3609,11 @@ public void testResetRunningTasks() throws Exception
         "id3",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3355,7 +3691,11 @@ public void testNoDataIngestionTasks()
         "id1",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3369,7 +3709,11 @@ public void testNoDataIngestionTasks()
         "id2",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3383,7 +3727,11 @@ public void testNoDataIngestionTasks()
         "id3",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3459,7 +3807,11 @@ public void testCheckpointForInactiveTaskGroup()
         "id1",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            topic,
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             topic,
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3473,7 +3825,11 @@ public void testCheckpointForInactiveTaskGroup()
         "id2",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            topic,
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             topic,
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3487,7 +3843,11 @@ public void testCheckpointForInactiveTaskGroup()
         "id3",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            topic,
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             topic,
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3517,8 +3877,10 @@ public void testCheckpointForInactiveTaskGroup()
     EasyMock.expect(taskStorage.getTask("id2")).andReturn(Optional.of(id2)).anyTimes();
     EasyMock.expect(taskStorage.getTask("id3")).andReturn(Optional.of(id3)).anyTimes();
     EasyMock.expect(
-        indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(new KafkaDataSourceMetadata(null)
-    ).anyTimes();
+                indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE))
+            .andReturn(new KafkaDataSourceMetadata(null)
+            )
+            .anyTimes();
     EasyMock.expect(taskClient.getStatusAsync("id1")).andReturn(Futures.immediateFuture(Status.READING));
     EasyMock.expect(taskClient.getStatusAsync("id2")).andReturn(Futures.immediateFuture(Status.READING));
     EasyMock.expect(taskClient.getStatusAsync("id3")).andReturn(Futures.immediateFuture(Status.READING));
@@ -3574,7 +3936,11 @@ public void testCheckpointForUnknownTaskGroup()
         "id1",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            topic,
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             topic,
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3588,7 +3954,11 @@ public void testCheckpointForUnknownTaskGroup()
         "id2",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            topic,
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             topic,
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3602,7 +3972,11 @@ public void testCheckpointForUnknownTaskGroup()
         "id3",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            topic,
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             topic,
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3624,8 +3998,10 @@ public void testCheckpointForUnknownTaskGroup()
     EasyMock.expect(taskStorage.getTask("id2")).andReturn(Optional.of(id2)).anyTimes();
     EasyMock.expect(taskStorage.getTask("id3")).andReturn(Optional.of(id3)).anyTimes();
     EasyMock.expect(
-        indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(new KafkaDataSourceMetadata(null)
-    ).anyTimes();
+                indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE))
+            .andReturn(new KafkaDataSourceMetadata(null)
+            )
+            .anyTimes();
 
     replayAll();
 
@@ -3703,7 +4079,11 @@ public void testSuspendedRunningTasks() throws Exception
         "id1",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 0L, 1, 0L, 2, 0L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3717,7 +4097,11 @@ public void testSuspendedRunningTasks() throws Exception
         "id2",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3731,7 +4115,11 @@ public void testSuspendedRunningTasks() throws Exception
         "id3",
         DATASOURCE,
         0,
-        new SeekableStreamStartSequenceNumbers<>("topic", singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L), ImmutableSet.of()),
+        new SeekableStreamStartSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, 10L, 1, 20L, 2, 30L),
+            ImmutableSet.of()
+        ),
         new SeekableStreamEndSequenceNumbers<>(
             "topic",
             singlePartitionMap(topic, 0, Long.MAX_VALUE, 1, Long.MAX_VALUE, 2, Long.MAX_VALUE)
@@ -3909,29 +4297,47 @@ public void testFailedInitializationAndRecovery() throws Exception
     Assert.assertEquals(topic, taskConfig.getStartSequenceNumbers().getStream());
     Assert.assertEquals(
         0L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 0))
+                  .longValue()
     );
     Assert.assertEquals(
         0L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 1))
+                  .longValue()
     );
     Assert.assertEquals(
         0L,
-        taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        taskConfig.getStartSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 2))
+                  .longValue()
     );
 
     Assert.assertEquals(topic, taskConfig.getEndSequenceNumbers().getStream());
     Assert.assertEquals(
         Long.MAX_VALUE,
-        taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 0)).longValue()
+        taskConfig.getEndSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 0))
+                  .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 1)).longValue()
+        taskConfig.getEndSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 1))
+                  .longValue()
     );
     Assert.assertEquals(
         Long.MAX_VALUE,
-        taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(new KafkaTopicPartition(false, topic, 2)).longValue()
+        taskConfig.getEndSequenceNumbers()
+                  .getPartitionSequenceNumberMap()
+                  .get(new KafkaTopicPartition(false, topic, 2))
+                  .longValue()
     );
   }
 
@@ -4147,7 +4553,10 @@ public void testKillIncompatibleTasks()
             singlePartitionMap(topic, 0, 0L, 2, 0L),
             ImmutableSet.of()
         ),
-        new SeekableStreamEndSequenceNumbers<>("topic", singlePartitionMap(topic, 0, Long.MAX_VALUE, 2, Long.MAX_VALUE)),
+        new SeekableStreamEndSequenceNumbers<>(
+            "topic",
+            singlePartitionMap(topic, 0, Long.MAX_VALUE, 2, Long.MAX_VALUE)
+        ),
         null,
         null,
         supervisor.getTuningConfig()
@@ -4485,67 +4894,80 @@ public void testResumeAllActivelyReadingTasks() throws Exception
     EasyMock.reset(taskClient);
     addSomeEvents(100);
 
-    KafkaIndexTask readingTask = createKafkaIndexTask("readingTask",
-                                                      DATASOURCE,
-                                                      0,
-                                                      new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 0L), Collections.emptySet()),
-                                                      new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 0, Long.MAX_VALUE)),
-                                                      null,
-                                                      null,
-                                                      supervisor.getTuningConfig()
-    );
-
-    KafkaIndexTask publishingTask = createKafkaIndexTask("publishingTask",
-                                                         DATASOURCE,
-                                                         1,
-                                                         new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 0L), Collections.emptySet()),
-                                                         new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 0, Long.MAX_VALUE)),
-                                                         null,
-                                                         null,
-                                                         supervisor.getTuningConfig()
-    );
-
-    KafkaIndexTask pausedTask = createKafkaIndexTask("pausedTask",
-                                                     DATASOURCE,
-                                                     1,
-                                                     new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 1, 0L), Collections.emptySet()),
-                                                     new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 1, Long.MAX_VALUE)),
-                                                     null,
-                                                     null,
-                                                     supervisor.getTuningConfig()
-    );
-
-    KafkaIndexTask failsToResumePausedTask = createKafkaIndexTask("failsToResumePausedTask",
-                                                                  DATASOURCE,
-                                                                  1,
-                                                                  new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 1, 0L), Collections.emptySet()),
-                                                                  new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 1, Long.MAX_VALUE)),
-                                                                  null,
-                                                                  null,
-                                                                  supervisor.getTuningConfig()
-    );
-
-    KafkaIndexTask waitingTask = createKafkaIndexTask("waitingTask",
-                                                      DATASOURCE,
-                                                      2,
-                                                      new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 2, 0L), Collections.emptySet()),
-                                                      new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 2, Long.MAX_VALUE)),
-                                                      null,
-                                                      null,
-                                                      supervisor.getTuningConfig()
-    );
-
-    KafkaIndexTask pendingTask = createKafkaIndexTask("pendingTask",
-                                                      DATASOURCE,
-                                                      2,
-                                                      new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 2, 0L), Collections.emptySet()),
-                                                      new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 2, Long.MAX_VALUE)),
-                                                      null,
-                                                      null,
-                                                      supervisor.getTuningConfig()
-    );
-
-    List<Task> tasks = ImmutableList.of(readingTask, publishingTask, pausedTask, failsToResumePausedTask, waitingTask, pendingTask);
+    KafkaIndexTask readingTask = createKafkaIndexTask(
+        "readingTask",
+        DATASOURCE,
+        0,
+        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 0L), Collections.emptySet()),
+        new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 0, Long.MAX_VALUE)),
+        null,
+        null,
+        supervisor.getTuningConfig()
+    );
+
+    KafkaIndexTask publishingTask = createKafkaIndexTask(
+        "publishingTask",
+        DATASOURCE,
+        1,
+        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 0, 0L), Collections.emptySet()),
+        new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 0, Long.MAX_VALUE)),
+        null,
+        null,
+        supervisor.getTuningConfig()
+    );
+
+    KafkaIndexTask pausedTask = createKafkaIndexTask(
+        "pausedTask",
+        DATASOURCE,
+        1,
+        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 1, 0L), Collections.emptySet()),
+        new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 1, Long.MAX_VALUE)),
+        null,
+        null,
+        supervisor.getTuningConfig()
+    );
+
+    KafkaIndexTask failsToResumePausedTask = createKafkaIndexTask(
+        "failsToResumePausedTask",
+        DATASOURCE,
+        1,
+        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 1, 0L), Collections.emptySet()),
+        new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 1, Long.MAX_VALUE)),
+        null,
+        null,
+        supervisor.getTuningConfig()
+    );
+
+    KafkaIndexTask waitingTask = createKafkaIndexTask(
+        "waitingTask",
+        DATASOURCE,
+        2,
+        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 2, 0L), Collections.emptySet()),
+        new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 2, Long.MAX_VALUE)),
+        null,
+        null,
+        supervisor.getTuningConfig()
+    );
+
+    KafkaIndexTask pendingTask = createKafkaIndexTask(
+        "pendingTask",
+        DATASOURCE,
+        2,
+        new SeekableStreamStartSequenceNumbers<>(topic, singlePartitionMap(topic, 2, 0L), Collections.emptySet()),
+        new SeekableStreamEndSequenceNumbers<>(topic, singlePartitionMap(topic, 2, Long.MAX_VALUE)),
+        null,
+        null,
+        supervisor.getTuningConfig()
+    );
+
+    List<Task> tasks = ImmutableList.of(
+        readingTask,
+        publishingTask,
+        pausedTask,
+        failsToResumePausedTask,
+        waitingTask,
+        pendingTask
+    );
     Collection taskRunnerWorkItems = ImmutableList.of(
         new TestTaskRunnerWorkItem(readingTask, null, TaskLocation.create("testHost", 1001, -1)),
         new TestTaskRunnerWorkItem(publishingTask, null, TaskLocation.create("testHost", 1002, -1)),
@@ -4641,7 +5063,8 @@ public void testResumeAllActivelyReadingTasks() throws Exception
     EasyMock.expect(taskClient.getStartTimeAsync(pausedTask.getId())).andReturn(Futures.immediateFuture(startTime));
     EasyMock.expect(taskClient.resumeAsync(pausedTask.getId())).andReturn(Futures.immediateFuture(true));
 
-    EasyMock.expect(taskClient.getStartTimeAsync(failsToResumePausedTask.getId())).andReturn(Futures.immediateFuture(startTime));
+    EasyMock.expect(taskClient.getStartTimeAsync(failsToResumePausedTask.getId()))
+            .andReturn(Futures.immediateFuture(startTime));
     EasyMock.expect(taskClient.resumeAsync(failsToResumePausedTask.getId())).andReturn(Futures.immediateFuture(false));
 
     Capture<String> shutdownTaskId = EasyMock.newCapture();
@@ -4856,11 +5279,10 @@ private TestableKafkaSupervisor getTestableSupervisor(
       public SeekableStreamIndexTaskClient<KafkaTopicPartition, Long> build(
           String dataSource,
           TaskInfoProvider taskInfoProvider,
-          int maxNumTasks,
-          SeekableStreamSupervisorTuningConfig tuningConfig
+          SeekableStreamSupervisorTuningConfig tuningConfig,
+          ScheduledExecutorService connectExec
       )
       {
-        Assert.assertEquals(replicas * taskCount, maxNumTasks);
         Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), tuningConfig.getHttpTimeout());
         Assert.assertEquals(TEST_CHAT_RETRIES, (long) tuningConfig.getChatRetries());
         return taskClient;
@@ -4971,11 +5393,10 @@ private TestableKafkaSupervisor getTestableSupervisorCustomIsTaskCurrent(
       public SeekableStreamIndexTaskClient<KafkaTopicPartition, Long> build(
           String dataSource,
           TaskInfoProvider taskInfoProvider,
-          int maxNumTasks,
-          SeekableStreamSupervisorTuningConfig tuningConfig
+          SeekableStreamSupervisorTuningConfig tuningConfig,
+          ScheduledExecutorService connectExec
       )
       {
-        Assert.assertEquals(replicas * taskCount, maxNumTasks);
         Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), tuningConfig.getHttpTimeout());
         Assert.assertEquals(TEST_CHAT_RETRIES, (long) tuningConfig.getChatRetries());
         return taskClient;
@@ -5089,11 +5510,10 @@ private KafkaSupervisor createSupervisor(
       public SeekableStreamIndexTaskClient<KafkaTopicPartition, Long> build(
           String dataSource,
           TaskInfoProvider taskInfoProvider,
-          int maxNumTasks,
-          SeekableStreamSupervisorTuningConfig tuningConfig
+          SeekableStreamSupervisorTuningConfig tuningConfig,
+          ScheduledExecutorService connectExec
       )
       {
-        Assert.assertEquals(replicas * taskCount, maxNumTasks);
         Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), tuningConfig.getHttpTimeout());
         Assert.assertEquals(TEST_CHAT_RETRIES, (long) tuningConfig.getChatRetries());
         return taskClient;
@@ -5227,30 +5647,63 @@ private KafkaIndexTask createKafkaIndexTask(
         OBJECT_MAPPER
     );
   }
-  
+
   private static ImmutableMap<KafkaTopicPartition, Long> singlePartitionMap(String topic, int partition, long offset)
   {
     return ImmutableMap.of(new KafkaTopicPartition(false, topic, partition), offset);
   }
 
-  private static ImmutableMap<KafkaTopicPartition, Long> singlePartitionMap(String topic, int partition1, long offset1, int partition2, long offset2)
+  private static ImmutableMap<KafkaTopicPartition, Long> singlePartitionMap(
+      String topic,
+      int partition1,
+      long offset1,
+      int partition2,
+      long offset2
+  )
   {
-    return ImmutableMap.of(new KafkaTopicPartition(false, topic, partition1), offset1, new KafkaTopicPartition(false, topic, partition2),
-      offset2);
+    return ImmutableMap.of(new KafkaTopicPartition(false, topic, partition1),
+                           offset1,
+                           new KafkaTopicPartition(false, topic, partition2),
+                           offset2
+    );
   }
 
-  private static ImmutableMap<KafkaTopicPartition, Long> singlePartitionMap(String topic, int partition1, long offset1,
-                                                              int partition2, long offset2, int partition3, long offset3)
+  private static ImmutableMap<KafkaTopicPartition, Long> singlePartitionMap(
+      String topic,
+      int partition1,
+      long offset1,
+      int partition2,
+      long offset2,
+      int partition3,
+      long offset3
+  )
   {
-    return ImmutableMap.of(new KafkaTopicPartition(false, topic, partition1), offset1, new KafkaTopicPartition(false, topic, partition2),
-                           offset2, new KafkaTopicPartition(false, topic, partition3), offset3);
+    return ImmutableMap.of(new KafkaTopicPartition(false, topic, partition1),
+                           offset1,
+                           new KafkaTopicPartition(false, topic, partition2),
+                           offset2,
+                           new KafkaTopicPartition(false, topic, partition3),
+                           offset3
+    );
   }
 
-  private static ImmutableMap<KafkaTopicPartition, Long> multiTopicPartitionMap(String topic, int partition1, long offset1,
-                                                                                int partition2, long offset2, int partition3, long offset3)
+  private static ImmutableMap<KafkaTopicPartition, Long> multiTopicPartitionMap(
+      String topic,
+      int partition1,
+      long offset1,
+      int partition2,
+      long offset2,
+      int partition3,
+      long offset3
+  )
   {
-    return ImmutableMap.of(new KafkaTopicPartition(true, topic, partition1), offset1, new KafkaTopicPartition(true, topic, partition2),
-        offset2, new KafkaTopicPartition(true, topic, partition3), offset3);
+    return ImmutableMap.of(new KafkaTopicPartition(true, topic, partition1),
+                           offset1,
+                           new KafkaTopicPartition(true, topic, partition2),
+                           offset2,
+                           new KafkaTopicPartition(true, topic, partition3),
+                           offset3
+    );
   }
 
   private static class TestTaskRunnerWorkItem extends TaskRunnerWorkItem
diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskClientFactory.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskClientFactory.java
index 2399008688c7..599c32b21584 100644
--- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskClientFactory.java
+++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskClientFactory.java
@@ -25,18 +25,18 @@
 import org.apache.druid.guice.annotations.EscalatedGlobal;
 import org.apache.druid.guice.annotations.Json;
 import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory;
-import org.apache.druid.rpc.ServiceClientFactory;
+import org.apache.druid.java.util.http.client.HttpClient;
 
 @LazySingleton
 public class KinesisIndexTaskClientFactory extends SeekableStreamIndexTaskClientFactory<String, String>
 {
   @Inject
   public KinesisIndexTaskClientFactory(
-      @EscalatedGlobal ServiceClientFactory serviceClientFactory,
+      @EscalatedGlobal HttpClient httpClient,
       @Json ObjectMapper mapper
   )
   {
-    super(serviceClientFactory, mapper);
+    super(httpClient, mapper);
   }
 
   @Override
diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java
index fe851a183a26..122a8e1c5ae8 100644
--- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java
+++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java
@@ -111,6 +111,7 @@
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
 
 public class KinesisSupervisorTest extends EasyMockSupport
 {
@@ -5117,11 +5118,10 @@ private TestableKinesisSupervisor getTestableSupervisor(
       public SeekableStreamIndexTaskClient<String, String> build(
           String dataSource,
           TaskInfoProvider taskInfoProvider,
-          int maxNumTasks,
-          SeekableStreamSupervisorTuningConfig tuningConfig
+          SeekableStreamSupervisorTuningConfig tuningConfig,
+          ScheduledExecutorService connectExec
       )
       {
-        Assert.assertEquals(replicas * taskCount, maxNumTasks);
         Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), tuningConfig.getHttpTimeout());
         Assert.assertEquals(TEST_CHAT_RETRIES, (long) tuningConfig.getChatRetries());
         return taskClient;
@@ -5259,14 +5259,10 @@ private TestableKinesisSupervisor getTestableSupervisor(
       public SeekableStreamIndexTaskClient<String, String> build(
           String dataSource,
           TaskInfoProvider taskInfoProvider,
-          int maxNumTasks,
-          SeekableStreamSupervisorTuningConfig tuningConfig
+          SeekableStreamSupervisorTuningConfig tuningConfig,
+          ScheduledExecutorService connectExec
       )
       {
-        Assert.assertEquals(
-            replicas * (autoScalerConfig != null ? autoScalerConfig.getTaskCountMax() : taskCount),
-            maxNumTasks
-        );
         Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), tuningConfig.getHttpTimeout());
         Assert.assertEquals(TEST_CHAT_RETRIES, (long) tuningConfig.getChatRetries());
         return taskClient;
@@ -5348,11 +5344,10 @@ private TestableKinesisSupervisor getTestableSupervisorCustomIsTaskCurrent(
       public SeekableStreamIndexTaskClient<String, String> build(
           String dataSource,
           TaskInfoProvider taskInfoProvider,
-          int maxNumTasks,
-          SeekableStreamSupervisorTuningConfig tuningConfig
+          SeekableStreamSupervisorTuningConfig tuningConfig,
+          ScheduledExecutorService connectExec
       )
       {
-        Assert.assertEquals(replicas * taskCount, maxNumTasks);
         Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), tuningConfig.getHttpTimeout());
         Assert.assertEquals(TEST_CHAT_RETRIES, (long) tuningConfig.getChatRetries());
         return taskClient;
@@ -5436,11 +5431,10 @@ private KinesisSupervisor createSupervisor(
       public SeekableStreamIndexTaskClient<String, String> build(
           String dataSource,
           TaskInfoProvider taskInfoProvider,
-          int maxNumTasks,
-          SeekableStreamSupervisorTuningConfig tuningConfig
+          SeekableStreamSupervisorTuningConfig tuningConfig,
+          ScheduledExecutorService connectExec
       )
       {
-        Assert.assertEquals(replicas * taskCount, maxNumTasks);
         Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), tuningConfig.getHttpTimeout());
         Assert.assertEquals(TEST_CHAT_RETRIES, (long) tuningConfig.getChatRetries());
         return taskClient;
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskClientFactory.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskClientFactory.java
index 5bdf8aaac390..c3b4e69f8327 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskClientFactory.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskClientFactory.java
@@ -21,31 +21,46 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.druid.indexing.common.TaskInfoProvider;
+import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor;
 import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorTuningConfig;
 import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.java.util.http.client.HttpClient;
 import org.apache.druid.rpc.ServiceClientFactory;
+import org.apache.druid.rpc.ServiceClientFactoryImpl;
+
+import java.util.concurrent.ScheduledExecutorService;
 
 public abstract class SeekableStreamIndexTaskClientFactory<PartitionIdType, SequenceOffsetType>
 {
   private static final Logger log = new Logger(SeekableStreamIndexTaskClientFactory.class);
 
-  private final ServiceClientFactory serviceClientFactory;
+  private final HttpClient httpClient;
   private final ObjectMapper jsonMapper;
 
   protected SeekableStreamIndexTaskClientFactory(
-      final ServiceClientFactory serviceClientFactory,
+      final HttpClient httpClient,
       final ObjectMapper jsonMapper
   )
   {
-    this.serviceClientFactory = serviceClientFactory;
+    this.httpClient = httpClient;
     this.jsonMapper = jsonMapper;
   }
 
+  /**
+   * Creates a task client for a specific supervisor.
+   *
+   * @param dataSource       task datasource
+   * @param taskInfoProvider task locator
+   * @param tuningConfig     from {@link SeekableStreamSupervisor#tuningConfig}
+   * @param connectExec      should generally be {@link SeekableStreamSupervisor#workerExec}. This is preferable to
+   *                         the global pool for the default {@link ServiceClientFactory}, to prevent callbacks from
+   *                         different supervisors from backlogging each other.
+   */
   public SeekableStreamIndexTaskClient<PartitionIdType, SequenceOffsetType> build(
       final String dataSource,
       final TaskInfoProvider taskInfoProvider,
-      final int maxNumTasks,
-      final SeekableStreamSupervisorTuningConfig tuningConfig
+      final SeekableStreamSupervisorTuningConfig tuningConfig,
+      final ScheduledExecutorService connectExec
   )
   {
     log.info(
@@ -57,7 +72,7 @@ public SeekableStreamIndexTaskClient<PartitionIdType, SequenceOffsetType> build(
 
     return new SeekableStreamIndexTaskClientAsyncImpl<PartitionIdType, SequenceOffsetType>(
         dataSource,
-        serviceClientFactory,
+        new ServiceClientFactoryImpl(httpClient, connectExec),
         taskInfoProvider,
         jsonMapper,
         tuningConfig.getHttpTimeout(),
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
index 86c4ba385bdd..4cbf0ccfa688 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
@@ -33,7 +33,7 @@
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.ListeningScheduledExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
@@ -65,7 +65,6 @@
 import org.apache.druid.indexing.seekablestream.SeekableStreamEndSequenceNumbers;
 import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTask;
 import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClient;
-import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientAsyncImpl;
 import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory;
 import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskIOConfig;
 import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner;
@@ -86,6 +85,7 @@
 import org.apache.druid.java.util.common.Stopwatch;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.concurrent.Execs;
+import org.apache.druid.java.util.common.concurrent.ScheduledExecutors;
 import org.apache.druid.java.util.emitter.EmittingLogger;
 import org.apache.druid.java.util.emitter.service.ServiceEmitter;
 import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
@@ -837,12 +837,25 @@ public String getType()
   private final SeekableStreamIndexTaskTuningConfig taskTuningConfig;
   private final String supervisorId;
   private final TaskInfoProvider taskInfoProvider;
-  private final long futureTimeoutInSeconds; // how long to wait for async operations to complete
   private final RowIngestionMetersFactory rowIngestionMetersFactory;
+  /**
+   * Single-threaded executor for running {@link Notice#handle()} from {@link #notices}.
+   */
   private final ExecutorService exec;
+  /**
+   * Single-threaded scheduled executor for adding periodic {@link RunNotice} to {@link #notices}.
+   */
   private final ScheduledExecutorService scheduledExec;
+  /**
+   * Single-threaded scheduled executor for reporting metircs on notice queue size, lag, etc.
+   * See {@link #scheduleReporting}.
+   */
   private final ScheduledExecutorService reportingExec;
-  private final ListeningExecutorService workerExec;
+  /**
+   * Multi-threaded executor for managing communications with workers, including handling callbacks from worker RPCs.
+   * Also serves as the connectExec for {@link #taskClient}.
+   */
+  private final ListeningScheduledExecutorService workerExec;
   private final NoticesQueue<Notice> notices = new NoticesQueue<>();
   private final Object stopLock = new Object();
   private final Object stateChangeLock = new Object();
@@ -903,21 +916,16 @@ public SeekableStreamSupervisor(
     );
 
     int workerThreads;
-    int maxNumTasks;
     if (autoScalerConfig != null && autoScalerConfig.getEnableTaskAutoScaler()) {
       log.info("Running Task autoscaler for datasource [%s]", dataSource);
 
       workerThreads = (this.tuningConfig.getWorkerThreads() != null
                        ? this.tuningConfig.getWorkerThreads()
                        : Math.min(10, autoScalerConfig.getTaskCountMax()));
-
-      maxNumTasks = autoScalerConfig.getTaskCountMax() * this.ioConfig.getReplicas();
     } else {
       workerThreads = (this.tuningConfig.getWorkerThreads() != null
                        ? this.tuningConfig.getWorkerThreads()
                        : Math.min(10, this.ioConfig.getTaskCount()));
-
-      maxNumTasks = this.ioConfig.getTaskCount() * this.ioConfig.getReplicas();
     }
 
     IdleConfig specIdleConfig = spec.getIoConfig().getIdleConfig();
@@ -938,7 +946,7 @@ public SeekableStreamSupervisor(
     }
 
     this.workerExec = MoreExecutors.listeningDecorator(
-        Execs.multiThreaded(
+        ScheduledExecutors.fixed(
             workerThreads,
             StringUtils.encodeForFormat(supervisorId) + "-Worker-%d"
         )
@@ -973,13 +981,7 @@ public Optional<TaskStatus> getTaskStatus(String id)
       }
     };
 
-    this.futureTimeoutInSeconds = Math.max(
-        MINIMUM_FUTURE_TIMEOUT_IN_SECONDS,
-        tuningConfig.getChatRetries() * (tuningConfig.getHttpTimeout().getStandardSeconds()
-                                         + SeekableStreamIndexTaskClientAsyncImpl.MAX_RETRY_WAIT_SECONDS)
-    );
-
-    this.taskClient = taskClientFactory.build(dataSource, taskInfoProvider, maxNumTasks, this.tuningConfig);
+    this.taskClient = taskClientFactory.build(dataSource, taskInfoProvider, this.tuningConfig, workerExec);
   }
 
   @Override
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java
index 40bbe84b623a..af66ce3b8b97 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java
@@ -179,7 +179,7 @@ public void setupTest()
     EasyMock.expect(taskClientFactory.build(
         EasyMock.anyString(),
         EasyMock.anyObject(),
-        EasyMock.anyInt(),
+        EasyMock.anyObject(),
         EasyMock.anyObject()
     )).andReturn(
         indexTaskClient).anyTimes();

From 43b325b6aaf3a96671ae81a64f727f262ade18eb Mon Sep 17 00:00:00 2001
From: Abhishek Radhakrishnan <abhishek.rb19@gmail.com>
Date: Tue, 22 Oct 2024 20:34:46 -0700
Subject: [PATCH 08/12] Add missing `@Nullable` annotations to SqlQuery
 (#17398)

---
 sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java b/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java
index 904d4c38715d..1364332aaa13 100644
--- a/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java
+++ b/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java
@@ -29,6 +29,7 @@
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.query.QueryContext;
 
+import javax.annotation.Nullable;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -57,12 +58,12 @@ public static List<TypedValue> getParameterList(List<SqlParameter> parameters)
   @JsonCreator
   public SqlQuery(
       @JsonProperty("query") final String query,
-      @JsonProperty("resultFormat") final ResultFormat resultFormat,
+      @JsonProperty("resultFormat") @Nullable final ResultFormat resultFormat,
       @JsonProperty("header") final boolean header,
       @JsonProperty("typesHeader") final boolean typesHeader,
       @JsonProperty("sqlTypesHeader") final boolean sqlTypesHeader,
-      @JsonProperty("context") final Map<String, Object> context,
-      @JsonProperty("parameters") final List<SqlParameter> parameters
+      @JsonProperty("context") @Nullable final Map<String, Object> context,
+      @JsonProperty("parameters") @Nullable final List<SqlParameter> parameters
   )
   {
     this.query = Preconditions.checkNotNull(query, "query");

From 1e96c85b383fdaff34faad7c3afbe3eb67c5d6c4 Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Wed, 23 Oct 2024 10:34:37 +0530
Subject: [PATCH 09/12] WindowOperatorQueryFrameProcessor: Avoid writing
 multiple frames to output channel in runIncrementally() (#17373)

WindowOperatorQueryFrameProcessor: Avoid writing multiple frames to output channel in runIncrementally()
---
 .../WindowOperatorQueryFrameProcessor.java    |  3 +
 ...WindowOperatorQueryFrameProcessorTest.java | 84 +++++++++++++++++++
 2 files changed, 87 insertions(+)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
index 6e91b19df4d4..04cdab3b1fe9 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessor.java
@@ -144,6 +144,9 @@ public ReturnOrAwait<Object> runIncrementally(IntSet readableInputs) throws IOEx
 
       if (needToProcessBatch()) {
         runAllOpsOnBatch();
+        if (inputChannel.isFinished()) {
+          return ReturnOrAwait.runAgain();
+        }
         flushAllRowsAndCols();
       }
       return ReturnOrAwait.runAgain();
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessorTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessorTest.java
index 02cb02360d93..5d1b350ca929 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessorTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/querykit/WindowOperatorQueryFrameProcessorTest.java
@@ -156,6 +156,90 @@ public void testFrameWriterReachingCapacity() throws IOException
     }
   }
 
+  @Test
+  public void testOutputChannelReachingCapacity() throws IOException
+  {
+    // This test validates that we don't end up writing multiple (2) frames to the output channel while reading from the input channel,
+    // in the scenario when the input channel has finished and receiver's completed() gets called.
+    final ReadableInput factChannel = buildWindowTestInputChannel();
+
+    RowSignature inputSignature = RowSignature.builder()
+                                              .add("cityName", ColumnType.STRING)
+                                              .add("added", ColumnType.LONG)
+                                              .build();
+
+    FrameReader frameReader = FrameReader.create(inputSignature);
+
+    RowSignature outputSignature = RowSignature.builder()
+                                               .addAll(inputSignature)
+                                               .add("w0", ColumnType.LONG)
+                                               .build();
+
+    final WindowOperatorQuery query = new WindowOperatorQuery(
+        new QueryDataSource(
+            Druids.newScanQueryBuilder()
+                  .dataSource(new TableDataSource("test"))
+                  .intervals(new LegacySegmentSpec(Intervals.ETERNITY))
+                  .columns("cityName", "added")
+                  .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
+                  .context(new HashMap<>())
+                  .build()),
+        new LegacySegmentSpec(Intervals.ETERNITY),
+        new HashMap<>(
+            // This ends up satisfying the criteria of needToProcessBatch() method,
+            // so we end up processing the rows we've read, hence writing the 1st frame to the output channel.
+            ImmutableMap.of(MultiStageQueryContext.MAX_ROWS_MATERIALIZED_IN_WINDOW, 12)
+        ),
+        outputSignature,
+        ImmutableList.of(
+            new WindowOperatorFactory(new WindowRowNumberProcessor("w0"))
+        ),
+        ImmutableList.of()
+    );
+
+    final FrameWriterFactory frameWriterFactory = new LimitedFrameWriterFactory(
+        FrameWriters.makeRowBasedFrameWriterFactory(
+            new ArenaMemoryAllocatorFactory(1 << 20),
+            outputSignature,
+            Collections.emptyList(),
+            false
+        ),
+        INPUT_ROWS.size() / 4 // This forces frameWriter's capacity to be reached, hence requiring another write.
+    );
+
+    final BlockingQueueFrameChannel outputChannel = BlockingQueueFrameChannel.minimal();
+    final WindowOperatorQueryFrameProcessor processor = new WindowOperatorQueryFrameProcessor(
+        query,
+        factChannel.getChannel(),
+        outputChannel.writable(),
+        frameWriterFactory,
+        frameReader,
+        new ObjectMapper(),
+        ImmutableList.of(
+            new WindowOperatorFactory(new WindowRowNumberProcessor("w0"))
+        )
+    );
+
+    exec.runFully(processor, null);
+
+    final Sequence<List<Object>> rowsFromProcessor = FrameTestUtil.readRowsFromFrameChannel(
+        outputChannel.readable(),
+        FrameReader.create(outputSignature)
+    );
+
+    List<List<Object>> outputRows = rowsFromProcessor.toList();
+    Assert.assertEquals(INPUT_ROWS.size(), outputRows.size());
+
+    for (int i = 0; i < INPUT_ROWS.size(); i++) {
+      Map<String, Object> inputRow = INPUT_ROWS.get(i);
+      List<Object> outputRow = outputRows.get(i);
+
+      Assert.assertEquals("cityName should match", inputRow.get("cityName"), outputRow.get(0));
+      Assert.assertEquals("added should match", inputRow.get("added"), outputRow.get(1));
+      Assert.assertEquals("row_number() should be correct", (long) i + 1, outputRow.get(2));
+    }
+  }
+
   @Test
   public void testProcessorRun() throws Exception
   {

From 7e8671caa99071d6b187c2665c40a2b84d404271 Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Wed, 23 Oct 2024 07:27:22 -0700
Subject: [PATCH 10/12] GroupByQueryConfig: Skip unnecessary toString. (#17396)

Calling toString on newConfig is unnecessary, because it will be done
automatically by the logger. This saves some effort under log levels
higher than DEBUG.
---
 .../java/org/apache/druid/query/groupby/GroupByQueryConfig.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryConfig.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryConfig.java
index dbd6f2869a75..23ef9fae820f 100644
--- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryConfig.java
+++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQueryConfig.java
@@ -370,7 +370,7 @@ public GroupByQueryConfig withOverrides(final GroupByQuery query)
         isMultiValueUnnestingEnabled()
     );
 
-    logger.debug("Override config for GroupBy query %s - %s", query.getId(), newConfig.toString());
+    logger.debug("Override config for GroupBy query %s - %s", query.getId(), newConfig);
     return newConfig;
   }
 

From c4b513e5990bd8e112a499be7a48ffddd74244ba Mon Sep 17 00:00:00 2001
From: Gian Merlino <gianmerlino@gmail.com>
Date: Thu, 24 Oct 2024 12:07:18 -0700
Subject: [PATCH 11/12] SeekableStreamSupervisor: Don't await task futures in
 workerExec. (#17403)

Following #17394, workerExec can get deadlocked with itself, because it
waits for task futures and is also used as the connectExec for the task
client. To fix this, we need to never await task futures in the workerExec.

There are two specific changes: in "verifyAndMergeCheckpoints" and
"checkpointTaskGroup", two "coalesceAndAwait" calls that formerly occurred
in workerExec are replaced with Futures.transform (using a callback in
workerExec).

Because this adjustment removes a source of blocking, it may also improve
supervisor responsiveness for high task counts. This is not the primary
goal, however. The primary goal is to fix the bug introduced by #17394.
---
 .../supervisor/SeekableStreamSupervisor.java  | 163 ++++++++++--------
 1 file changed, 90 insertions(+), 73 deletions(-)

diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
index 4cbf0ccfa688..8b4845b08d07 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java
@@ -31,6 +31,7 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.AsyncFunction;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningScheduledExecutorService;
@@ -852,7 +853,7 @@ public String getType()
    */
   private final ScheduledExecutorService reportingExec;
   /**
-   * Multi-threaded executor for managing communications with workers, including handling callbacks from worker RPCs.
+   * Multi-threaded executor for callbacks from worker RPCs.
    * Also serves as the connectExec for {@link #taskClient}.
    */
   private final ListeningScheduledExecutorService workerExec;
@@ -2314,7 +2315,7 @@ private void verifyAndMergeCheckpoints(final Collection<TaskGroup> taskGroupsToV
     final List<ListenableFuture<Object>> futures = new ArrayList<>();
     for (TaskGroup taskGroup : taskGroupsToVerify) {
       //noinspection unchecked
-      futures.add((ListenableFuture<Object>) workerExec.submit(() -> verifyAndMergeCheckpoints(taskGroup)));
+      futures.add((ListenableFuture<Object>) verifyAndMergeCheckpoints(taskGroup));
     }
 
     try {
@@ -2327,16 +2328,11 @@ private void verifyAndMergeCheckpoints(final Collection<TaskGroup> taskGroupsToV
   }
 
   /**
-   * This method does two things -
-   * 1. Makes sure the checkpoints information in the taskGroup is consistent with that of the tasks, if not kill
-   * inconsistent tasks.
-   * 2. truncates the checkpoints in the taskGroup corresponding to which segments have been published, so that any newly
-   * created tasks for the taskGroup start indexing from after the latest published sequences.
+   * Calls {@link SeekableStreamIndexTaskClient#getCheckpointsAsync(String, boolean)} on each task in the group,
+   * then calls {@link #verifyAndMergeCheckpoints(TaskGroup, List, List)} as a callback in {@link #workerExec}.
    */
-  private void verifyAndMergeCheckpoints(final TaskGroup taskGroup)
+  private ListenableFuture<?> verifyAndMergeCheckpoints(final TaskGroup taskGroup)
   {
-    final int groupId = taskGroup.groupId;
-    final List<Pair<String, TreeMap<Integer, Map<PartitionIdType, SequenceOffsetType>>>> taskSequences = new ArrayList<>();
     final List<ListenableFuture<TreeMap<Integer, Map<PartitionIdType, SequenceOffsetType>>>> futures = new ArrayList<>();
     final List<String> taskIds = new ArrayList<>();
 
@@ -2350,30 +2346,48 @@ private void verifyAndMergeCheckpoints(final TaskGroup taskGroup)
       taskIds.add(taskId);
     }
 
-    try {
-      List<Either<Throwable, TreeMap<Integer, Map<PartitionIdType, SequenceOffsetType>>>> futuresResult =
-          coalesceAndAwait(futures);
-
-      for (int i = 0; i < futuresResult.size(); i++) {
-        final Either<Throwable, TreeMap<Integer, Map<PartitionIdType, SequenceOffsetType>>> futureResult =
-            futuresResult.get(i);
-        final String taskId = taskIds.get(i);
-        if (futureResult.isError()) {
-          final Throwable e = new RuntimeException(futureResult.error());
-          stateManager.recordThrowableEvent(e);
-          log.error(e, "Problem while getting checkpoints for task [%s], killing the task", taskId);
-          killTask(taskId, "Exception[%s] while getting checkpoints", e.getClass());
-          taskGroup.tasks.remove(taskId);
-        } else if (futureResult.valueOrThrow().isEmpty()) {
-          log.warn("Ignoring task [%s], as probably it is not started running yet", taskId);
-        } else {
-          taskSequences.add(new Pair<>(taskId, futureResult.valueOrThrow()));
-        }
+    return Futures.transform(
+        FutureUtils.coalesce(futures),
+        futuresResult -> {
+          verifyAndMergeCheckpoints(taskGroup, taskIds, futuresResult);
+          return null;
+        },
+        workerExec
+    );
+  }
+
+  /**
+   * This method does two things in {@link #workerExec} -
+   * 1. Makes sure the checkpoints information in the taskGroup is consistent with that of the tasks, if not kill
+   * inconsistent tasks.
+   * 2. truncates the checkpoints in the taskGroup corresponding to which segments have been published, so that any newly
+   * created tasks for the taskGroup start indexing from after the latest published sequences.
+   */
+  private void verifyAndMergeCheckpoints(
+      final TaskGroup taskGroup,
+      final List<String> taskIds,
+      final List<Either<Throwable, TreeMap<Integer, Map<PartitionIdType, SequenceOffsetType>>>> checkpointResults
+  )
+  {
+    final int groupId = taskGroup.groupId;
+    final List<Pair<String, TreeMap<Integer, Map<PartitionIdType, SequenceOffsetType>>>> taskSequences = new ArrayList<>();
+
+    for (int i = 0; i < checkpointResults.size(); i++) {
+      final Either<Throwable, TreeMap<Integer, Map<PartitionIdType, SequenceOffsetType>>> checkpointResult =
+          checkpointResults.get(i);
+      final String taskId = taskIds.get(i);
+      if (checkpointResult.isError()) {
+        final Throwable e = new RuntimeException(checkpointResult.error());
+        stateManager.recordThrowableEvent(e);
+        log.error(e, "Problem while getting checkpoints for task [%s], killing the task", taskId);
+        killTask(taskId, "Exception[%s] while getting checkpoints", e.getClass());
+        taskGroup.tasks.remove(taskId);
+      } else if (checkpointResult.valueOrThrow().isEmpty()) {
+        log.warn("Ignoring task [%s], as probably it is not started running yet", taskId);
+      } else {
+        taskSequences.add(new Pair<>(taskId, checkpointResult.valueOrThrow()));
       }
     }
-    catch (Exception e) {
-      throw new RuntimeException(e);
-    }
 
     final DataSourceMetadata rawDataSourceMetadata = indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(
         dataSource);
@@ -3361,13 +3375,12 @@ public Map<PartitionIdType, SequenceOffsetType> apply(@Nullable Object input)
       pauseFutures.add(taskClient.pauseAsync(taskId));
     }
 
-    return Futures.transform(
+    return Futures.transformAsync(
         FutureUtils.coalesce(pauseFutures),
-        new Function<List<Either<Throwable, Map<PartitionIdType, SequenceOffsetType>>>, Map<PartitionIdType, SequenceOffsetType>>()
+        new AsyncFunction<List<Either<Throwable, Map<PartitionIdType, SequenceOffsetType>>>, Map<PartitionIdType, SequenceOffsetType>>()
         {
-          @Nullable
           @Override
-          public Map<PartitionIdType, SequenceOffsetType> apply(List<Either<Throwable, Map<PartitionIdType, SequenceOffsetType>>> input)
+          public ListenableFuture<Map<PartitionIdType, SequenceOffsetType>> apply(List<Either<Throwable, Map<PartitionIdType, SequenceOffsetType>>> input)
           {
             // 3) Build a map of the highest sequence read by any task in the group for each partition
             final Map<PartitionIdType, SequenceOffsetType> endOffsets = new HashMap<>();
@@ -3408,50 +3421,54 @@ public Map<PartitionIdType, SequenceOffsetType> apply(List<Either<Throwable, Map
 
             if (setEndOffsetTaskIds.isEmpty()) {
               log.info("All tasks in taskGroup [%d] have failed, tasks will be re-created", taskGroup.groupId);
-              return null;
+              return Futures.immediateFuture(null);
             }
 
-            try {
-
-              if (endOffsets.equals(taskGroup.checkpointSequences.lastEntry().getValue())) {
-                log.warn(
-                    "Checkpoint[%s] is same as the start sequences[%s] of latest sequence for the taskGroup[%d].",
-                    endOffsets,
-                    taskGroup.checkpointSequences.lastEntry().getValue(),
-                    taskGroup.groupId
-                );
-              }
-
-              log.info(
-                  "Setting endOffsets for tasks in taskGroup[%d] to [%s]",
-                  taskGroup.groupId, endOffsets
+            if (endOffsets.equals(taskGroup.checkpointSequences.lastEntry().getValue())) {
+              log.warn(
+                  "Checkpoint[%s] is same as the start sequences[%s] of latest sequence for the taskGroup[%d].",
+                  endOffsets,
+                  taskGroup.checkpointSequences.lastEntry().getValue(),
+                  taskGroup.groupId
               );
-              for (final String taskId : setEndOffsetTaskIds) {
-                setEndOffsetFutures.add(taskClient.setEndOffsetsAsync(taskId, endOffsets, finalize));
-              }
-
-              List<Either<Throwable, Boolean>> results = coalesceAndAwait(setEndOffsetFutures);
-              for (int i = 0; i < results.size(); i++) {
-                if (results.get(i).isValue() && Boolean.valueOf(true).equals(results.get(i).valueOrThrow())) {
-                  log.info("Successfully set endOffsets for task[%s] and resumed it", setEndOffsetTaskIds.get(i));
-                } else {
-                  String taskId = setEndOffsetTaskIds.get(i);
-                  killTask(taskId, "Failed to set end offsets, killing task");
-                  taskGroup.tasks.remove(taskId);
-                }
-              }
-            }
-            catch (Exception e) {
-              log.error("An exception occurred while setting end offsets: [%s]", e.getMessage());
-              throw new RuntimeException(e);
             }
 
-            if (taskGroup.tasks.isEmpty()) {
-              log.info("All tasks in taskGroup[%d] have failed, tasks will be re-created", taskGroup.groupId);
-              return null;
+            log.info(
+                "Setting endOffsets for tasks in taskGroup[%d] to [%s]",
+                taskGroup.groupId, endOffsets
+            );
+            for (final String taskId : setEndOffsetTaskIds) {
+              setEndOffsetFutures.add(taskClient.setEndOffsetsAsync(taskId, endOffsets, finalize));
             }
 
-            return endOffsets;
+            return Futures.transform(
+                FutureUtils.coalesce(setEndOffsetFutures),
+                results -> {
+                  try {
+                    for (int i = 0; i < results.size(); i++) {
+                      if (results.get(i).isValue() && Boolean.valueOf(true).equals(results.get(i).valueOrThrow())) {
+                        log.info("Successfully set endOffsets for task[%s] and resumed it", setEndOffsetTaskIds.get(i));
+                      } else {
+                        String taskId = setEndOffsetTaskIds.get(i);
+                        killTask(taskId, "Failed to set end offsets, killing task");
+                        taskGroup.tasks.remove(taskId);
+                      }
+                    }
+                  }
+                  catch (Exception e) {
+                    log.error("An exception occurred while setting end offsets: [%s]", e.getMessage());
+                    throw new RuntimeException(e);
+                  }
+
+                  if (taskGroup.tasks.isEmpty()) {
+                    log.info("All tasks in taskGroup[%d] have failed, tasks will be re-created", taskGroup.groupId);
+                    return null;
+                  }
+
+                  return endOffsets;
+                },
+                workerExec
+            );
           }
         },
         workerExec

From fe0f4150c9a0156a2d95034fd8f626dfc552b34b Mon Sep 17 00:00:00 2001
From: Akshat Jain <akjn11@gmail.com>
Date: Fri, 25 Oct 2024 15:02:32 +0530
Subject: [PATCH 12/12] MSQ ingestion: Improve error message on encountering
 non-long timestamp column (#17411)

This PR improves the error message during MSQ ingestion if we encounter a non-long timestamp column.
---
 .../org/apache/druid/msq/exec/MSQTasks.java   |  8 +++-
 .../apache/druid/msq/exec/MSQTasksTest.java   | 39 +++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/MSQTasks.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/MSQTasks.java
index 2dff4419bdbc..75dc1271e3a2 100644
--- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/MSQTasks.java
+++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/MSQTasks.java
@@ -114,7 +114,13 @@ public static long primaryTimestampFromObjectForInsert(final Object timestamp)
       // be a long at execution time. So a nice user-friendly message isn't needed here: it would only happen
       // if the SQL layer is bypassed. Nice, friendly users wouldn't do that :)
       final UnknownFault fault =
-          UnknownFault.forMessage(StringUtils.format("Incorrect type for [%s]", ColumnHolder.TIME_COLUMN_NAME));
+          UnknownFault.forMessage(
+              StringUtils.format(
+                  "Incorrect type for column [%s]. Expected LONG but got type [%s]. Please ensure that the value is cast to LONG.",
+                  ColumnHolder.TIME_COLUMN_NAME,
+                  timestamp.getClass().getSimpleName()
+              )
+          );
       throw new MSQException(fault);
     }
   }
diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQTasksTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQTasksTest.java
index 904510408ff0..eec5e9d4ebe4 100644
--- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQTasksTest.java
+++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQTasksTest.java
@@ -34,8 +34,10 @@
 import org.apache.druid.indexer.TaskStatusPlus;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.msq.indexing.MSQWorkerTask;
 import org.apache.druid.msq.indexing.MSQWorkerTaskLauncher;
+import org.apache.druid.msq.indexing.error.InsertTimeNullFault;
 import org.apache.druid.msq.indexing.error.MSQErrorReport;
 import org.apache.druid.msq.indexing.error.MSQException;
 import org.apache.druid.msq.indexing.error.MSQFaultUtils;
@@ -46,6 +48,7 @@
 import org.apache.druid.msq.indexing.error.TooManyWorkersFault;
 import org.apache.druid.msq.indexing.error.UnknownFault;
 import org.apache.druid.msq.indexing.error.WorkerRpcFailedFault;
+import org.apache.druid.segment.column.ColumnHolder;
 import org.apache.druid.utils.CollectionUtils;
 import org.junit.Assert;
 import org.junit.Test;
@@ -244,6 +247,42 @@ public void test_queryWithoutEnoughSlots_shouldThrowException()
     }
   }
 
+  @Test
+  public void test_getPrimaryTimestampFromObjectForInsert_longValue()
+  {
+    Assert.assertEquals(100, MSQTasks.primaryTimestampFromObjectForInsert(100L));
+  }
+
+  @Test
+  public void test_getPrimaryTimestampFromObjectForInsert_nullValueShouldThrowError()
+  {
+    final MSQException e = Assert.assertThrows(
+        MSQException.class,
+        () -> MSQTasks.primaryTimestampFromObjectForInsert(null)
+    );
+    Assert.assertEquals(InsertTimeNullFault.INSTANCE, e.getFault());
+  }
+
+  @Test
+  public void test_getPrimaryTimestampFromObjectForInsert_DoubleValueShouldThrowError()
+  {
+    final Object timestamp = 1.693837200123456E15;
+    final MSQException e = Assert.assertThrows(
+        MSQException.class,
+        () -> MSQTasks.primaryTimestampFromObjectForInsert(timestamp)
+    );
+    Assert.assertEquals(
+        UnknownFault.forMessage(
+            StringUtils.format(
+                "Incorrect type for column [%s]. Expected LONG but got type [%s]. Please ensure that the value is cast to LONG.",
+                ColumnHolder.TIME_COLUMN_NAME,
+                timestamp.getClass().getSimpleName()
+            )
+        ),
+        e.getFault()
+    );
+  }
+
   static class TasksTestOverlordClient extends NoopOverlordClient
   {
     // Num of slots available for tasks