Compaction with MSQ

indexing-service/pom.xml

@@ -48,6 +48,11 @@
             <artifactId>druid-indexing-hadoop</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.druid</groupId>
+            <artifactId>druid-sql</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
             <artifactId>metrics-core</artifactId>

indexing-service/src/main/java/org/apache/druid/indexing/common/TaskToolbox.java

@@ -45,6 +45,7 @@
 import org.apache.druid.query.QueryProcessingPool;
 import org.apache.druid.query.QueryRunnerFactoryConglomerate;
 import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.rpc.indexing.RouterClient;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.IndexMergerV9;
 import org.apache.druid.segment.handoff.SegmentHandoffNotifierFactory;
@@ -120,6 +121,7 @@ public class TaskToolbox
   private final AppenderatorsManager appenderatorsManager;
   private final OverlordClient overlordClient;
   private final CoordinatorClient coordinatorClient;
+  private final RouterClient routerClient;
 
   // Used by only native parallel tasks
   private final IntermediaryDataManager intermediaryDataManager;
@@ -165,6 +167,7 @@ public TaskToolbox(
       AppenderatorsManager appenderatorsManager,
       OverlordClient overlordClient,
       CoordinatorClient coordinatorClient,
+      RouterClient routerClient,
       ParallelIndexSupervisorTaskClientProvider supervisorTaskClientProvider,
       ShuffleClient shuffleClient,
       TaskLogPusher taskLogPusher,
@@ -207,6 +210,7 @@ public TaskToolbox(
     this.appenderatorsManager = appenderatorsManager;
     this.overlordClient = overlordClient;
     this.coordinatorClient = coordinatorClient;
+    this.routerClient = routerClient;
     this.supervisorTaskClientProvider = supervisorTaskClientProvider;
     this.shuffleClient = shuffleClient;
     this.taskLogPusher = taskLogPusher;
@@ -448,6 +452,11 @@ public CoordinatorClient getCoordinatorClient()
     return coordinatorClient;
   }
 
+  public RouterClient getRouterClient()
+  {
+    return routerClient;
+  }
+
   public ParallelIndexSupervisorTaskClientProvider getSupervisorTaskClientProvider()
   {
     return supervisorTaskClientProvider;
@@ -538,6 +547,7 @@ public static class Builder
     private AppenderatorsManager appenderatorsManager;
     private OverlordClient overlordClient;
     private CoordinatorClient coordinatorClient;
+    private RouterClient routerClient;
     private IntermediaryDataManager intermediaryDataManager;
     private ParallelIndexSupervisorTaskClientProvider supervisorTaskClientProvider;
     private ShuffleClient shuffleClient;
@@ -584,6 +594,7 @@ public Builder(TaskToolbox other)
       this.appenderatorsManager = other.appenderatorsManager;
       this.overlordClient = other.overlordClient;
       this.coordinatorClient = other.coordinatorClient;
+      this.routerClient = other.routerClient;
       this.intermediaryDataManager = other.intermediaryDataManager;
       this.supervisorTaskClientProvider = other.supervisorTaskClientProvider;
       this.shuffleClient = other.shuffleClient;
@@ -793,6 +804,12 @@ public Builder coordinatorClient(final CoordinatorClient coordinatorClient)
       return this;
     }
 
+    public Builder routerClient(final RouterClient routerClient)
+    {
+      this.routerClient = routerClient;
+      return this;
+    }
+
     public Builder intermediaryDataManager(final IntermediaryDataManager intermediaryDataManager)
     {
       this.intermediaryDataManager = intermediaryDataManager;
@@ -861,6 +878,7 @@ public TaskToolbox build()
           appenderatorsManager,
           overlordClient,
           coordinatorClient,
+          routerClient,
           supervisorTaskClientProvider,
           shuffleClient,
           taskLogPusher,

indexing-service/src/main/java/org/apache/druid/indexing/common/TaskToolboxFactory.java

@@ -47,6 +47,7 @@
 import org.apache.druid.query.QueryRunnerFactoryConglomerate;
 import org.apache.druid.rpc.StandardRetryPolicy;
 import org.apache.druid.rpc.indexing.OverlordClient;
+import org.apache.druid.rpc.indexing.RouterClient;
 import org.apache.druid.segment.IndexIO;
 import org.apache.druid.segment.IndexMergerV9Factory;
 import org.apache.druid.segment.handoff.SegmentHandoffNotifierFactory;
@@ -105,6 +106,7 @@ public class TaskToolboxFactory
   private final AppenderatorsManager appenderatorsManager;
   private final OverlordClient overlordClient;
   private final CoordinatorClient coordinatorClient;
+  private final RouterClient routerClient;
 
   // Used by only native parallel tasks
   private final IntermediaryDataManager intermediaryDataManager;
@@ -149,6 +151,7 @@ public TaskToolboxFactory(
       AppenderatorsManager appenderatorsManager,
       OverlordClient overlordClient,
       CoordinatorClient coordinatorClient,
+      RouterClient routerClient,
       ParallelIndexSupervisorTaskClientProvider supervisorTaskClientProvider,
       ShuffleClient shuffleClient,
       TaskLogPusher taskLogPusher,
@@ -189,6 +192,7 @@ public TaskToolboxFactory(
     this.appenderatorsManager = appenderatorsManager;
     this.overlordClient = overlordClient;
     this.coordinatorClient = coordinatorClient;
+    this.routerClient = routerClient;
     this.supervisorTaskClientProvider = supervisorTaskClientProvider;
     this.shuffleClient = shuffleClient;
     this.taskLogPusher = taskLogPusher;
@@ -251,6 +255,7 @@ public TaskToolbox build(TaskConfig config, Task task)
         // Calls will still eventually fail if problems persist.
         .overlordClient(overlordClient.withRetryPolicy(StandardRetryPolicy.aboutAnHour()))
         .coordinatorClient(coordinatorClient.withRetryPolicy(StandardRetryPolicy.aboutAnHour()))
+        .routerClient(routerClient)
         .supervisorTaskClientProvider(supervisorTaskClientProvider)
         .shuffleClient(shuffleClient)
         .taskLogPusher(taskLogPusher)