Pull Dependencies only once

distribution/pom.xml

@@ -184,6 +184,9 @@
                                         <argument>-Ddruid.extensions.loadList=[]</argument>
                                         <argument>-Ddruid.extensions.directory=${project.build.directory}/extensions
                                         </argument>
+                                        <argument>
+                                            -Ddruid.extensions.commonDependenciesDir=${project.build.directory}/commonDeps
+                                        </argument>
                                         <argument>
                                             -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
                                         </argument>
@@ -385,6 +388,9 @@
                                         <argument>-Ddruid.extensions.loadList=[]</argument>
                                         <argument>-Ddruid.extensions.directory=${project.build.directory}/extensions
                                         </argument>
+                                        <argument>
+                                            -Ddruid.extensions.commonDependenciesDir=${project.build.directory}/commonDeps
+                                        </argument>
                                         <argument>
                                             -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
                                         </argument>
@@ -493,6 +499,9 @@
                                         <argument>-Ddruid.extensions.loadList=[]</argument>
                                         <argument>-Ddruid.extensions.directory=${project.build.directory}/extensions
                                         </argument>
+                                        <argument>
+                                            -Ddruid.extensions.commonDependenciesDir=${project.build.directory}/commonDeps
+                                        </argument>
                                         <argument>
                                             -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
                                         </argument>

distribution/src/assembly/assembly.xml

@@ -34,6 +34,15 @@
             <outputDirectory>extensions</outputDirectory>
         </fileSet>
 
+        <fileSet>
+            <directory>${project.build.directory}/commonDeps</directory>
+            <includes>
+                <include>*</include>
+            </includes>
+            <outputDirectory>lib</outputDirectory>
+        </fileSet>
+
+
         <fileSet>
             <directory>${project.build.directory}/hadoop-dependencies</directory>
             <includes>

distribution/src/assembly/integration-test-assembly.xml

@@ -35,6 +35,14 @@
             <outputDirectory>extensions</outputDirectory>
         </fileSet>
 
+        <fileSet>
+            <directory>${project.build.directory}/commonDeps</directory>
+            <includes>
+                <include>*</include>
+            </includes>
+            <outputDirectory>lib</outputDirectory>
+        </fileSet>
+
         <fileSet>
             <directory>${project.build.directory}/hadoop-dependencies</directory>
             <includes>

pom.xml

@@ -1873,7 +1873,7 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-assembly-plugin</artifactId>
-                    <version>3.1.0</version>
+                    <version>3.4.0</version>
                 </plugin>
                 <plugin>
                     <groupId>org.codehaus.mojo</groupId>

processing/src/main/java/org/apache/druid/guice/ExtensionsConfig.java

@@ -46,6 +46,9 @@ public class ExtensionsConfig
   @JsonProperty
   private String hadoopContainerDruidClasspath = null;
 
+  @JsonProperty
+  private String commonDependenciesDir = null;
+
   //Only applicable when hadoopContainerDruidClasspath is explicitly specified.
   @JsonProperty
   private boolean addExtensionsToHadoopContainer = false;
@@ -73,6 +76,11 @@ public String getHadoopDependenciesDir()
     return hadoopDependenciesDir;
   }
 
+  public String getCommonDependenciesDir()
+  {
+    return commonDependenciesDir;
+  }
+
   public String getHadoopContainerDruidClasspath()
   {
     return hadoopContainerDruidClasspath;

services/src/main/java/org/apache/druid/cli/PullDependencies.java

@@ -62,6 +62,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -78,6 +79,8 @@ public class PullDependencies implements Runnable
       "https://repo1.maven.org/maven2/"
   );
 
+  private static final HashMap<Artifact, File> ARTIFACT_FILE_HASH_MAP = new HashMap<>();
+
   private static final Dependencies PROVIDED_BY_CORE_DEPENDENCIES =
       Dependencies.builder()
                   .put("com.squareup.okhttp", "okhttp")
@@ -260,14 +263,24 @@ public void run()
 
     final File extensionsDir = new File(extensionsConfig.getDirectory());
     final File hadoopDependenciesDir = new File(extensionsConfig.getHadoopDependenciesDir());
+    File commonDependenciesDir = null;
+    if (extensionsConfig.getCommonDependenciesDir() != null) {
+      commonDependenciesDir = new File(extensionsConfig.getCommonDependenciesDir());
+    }
 
     try {
       if (clean) {
         FileUtils.deleteDirectory(extensionsDir);
         FileUtils.deleteDirectory(hadoopDependenciesDir);
+        if (commonDependenciesDir != null) {
+          FileUtils.deleteDirectory(commonDependenciesDir);
+        }
       }
       FileUtils.mkdirp(extensionsDir);
       FileUtils.mkdirp(hadoopDependenciesDir);
+      if (commonDependenciesDir != null) {
+        FileUtils.mkdirp(commonDependenciesDir);
+      }
     }
     catch (IOException e) {
       log.error(e, "Unable to clear or create extension directory at [%s]", extensionsDir);
@@ -289,7 +302,7 @@ public void run()
         File currExtensionDir = new File(extensionsDir, versionedArtifact.getArtifactId());
         createExtensionDirectory(coordinate, currExtensionDir);
 
-        downloadExtension(versionedArtifact, currExtensionDir);
+        downloadExtension(versionedArtifact, currExtensionDir, commonDependenciesDir);
       }
       log.info("Finish downloading dependencies for extension coordinates: [%s]", coordinates);
 
@@ -308,7 +321,7 @@ public void run()
         currExtensionDir = new File(currExtensionDir, versionedArtifact.getVersion());
         createExtensionDirectory(hadoopCoordinate, currExtensionDir);
 
-        downloadExtension(versionedArtifact, currExtensionDir, hadoopExclusions);
+        downloadExtension(versionedArtifact, currExtensionDir, hadoopExclusions, null);
       }
       log.info("Finish downloading dependencies for hadoop extension coordinates: [%s]", hadoopCoordinates);
     }
@@ -341,12 +354,12 @@ protected Artifact getArtifact(String coordinate)
    * @param versionedArtifact The maven artifact of the extension
    * @param toLocation        The location where this extension will be downloaded to
    */
-  private void downloadExtension(Artifact versionedArtifact, File toLocation)
+  private void downloadExtension(Artifact versionedArtifact, File toLocation, File commonDependenciesDir)
   {
-    downloadExtension(versionedArtifact, toLocation, PROVIDED_BY_CORE_DEPENDENCIES);
+    downloadExtension(versionedArtifact, toLocation, PROVIDED_BY_CORE_DEPENDENCIES, commonDependenciesDir);
   }
 
-  private void downloadExtension(Artifact versionedArtifact, File toLocation, Dependencies exclusions)
+  private void downloadExtension(Artifact versionedArtifact, File toLocation, Dependencies exclusions, File commonDependenciesDir)
   {
     final CollectRequest collectRequest = new CollectRequest();
     collectRequest.setRoot(new Dependency(versionedArtifact, JavaScopes.RUNTIME));
@@ -400,8 +413,14 @@ private void downloadExtension(Artifact versionedArtifact, File toLocation, Depe
       for (Artifact artifact : artifacts) {
         if (exclusions.contain(artifact)) {
           log.debug("Skipped Artifact[%s]", artifact);
+        } else if (commonDependenciesDir != null && ARTIFACT_FILE_HASH_MAP.containsKey(artifact)) {
+          log.info("Copying file [%s] to common dependencies directory [%s]", artifact.getFile().getName(), commonDependenciesDir.getAbsolutePath());
+          org.apache.commons.io.FileUtils.copyFileToDirectory(artifact.getFile(), commonDependenciesDir);
+          org.apache.commons.io.FileUtils.deleteQuietly(ARTIFACT_FILE_HASH_MAP.get(artifact));
         } else {
           log.info("Adding file [%s] at [%s]", artifact.getFile().getName(), toLocation.getAbsolutePath());
+          File file = new File(toLocation, artifact.getFile().getName());
+          ARTIFACT_FILE_HASH_MAP.put(artifact, file);
           org.apache.commons.io.FileUtils.copyFileToDirectory(artifact.getFile(), toLocation);
         }
       }

services/src/test/java/org/apache/druid/cli/PullDependenciesTest.java

@@ -100,26 +100,34 @@ public class PullDependenciesTest
   private final Artifact extension_B = new DefaultArtifact(EXTENSION_B_COORDINATE);
   private final Artifact hadoop_client_2_3_0 = new DefaultArtifact(HADOOP_CLIENT_2_3_0_COORDINATE);
   private final Artifact hadoop_client_2_4_0 = new DefaultArtifact(HADOOP_CLIENT_2_4_0_COORDINATE);
+  private List<String> commonDependencyList;
+  private List<String> extension_A_expected;
+  private List<String> extension_B_expected;
   private PullDependencies pullDependencies;
   private File rootExtensionsDir;
   private File rootHadoopDependenciesDir;
+  private File rootCommonDependenciesDir;
 
   @Before
   public void setUp() throws Exception
   {
     localRepo = temporaryFolder.newFolder("local_repo");
     extensionToDependency = new HashMap<>();
 
-    extensionToDependency.put(extension_A, ImmutableList.of("a", "b", "c"));
-    extensionToDependency.put(extension_B, ImmutableList.of("d", "e"));
+    extensionToDependency.put(extension_A, ImmutableList.of("a", "b", "c", "d"));
+    extensionToDependency.put(extension_B, ImmutableList.of("c", "d", "e"));
     extensionToDependency.put(hadoop_client_2_3_0, ImmutableList.of("f", "g"));
     extensionToDependency.put(
         hadoop_client_2_4_0,
         ImmutableList.of("h", "i", HADOOP_CLIENT_VULNERABLE_ARTIFACTID1, HADOOP_CLIENT_VULNERABLE_ARTIFACTID2)
     );
+    commonDependencyList = ImmutableList.of("c", "d");
+    extension_A_expected = ImmutableList.of("a", "b");
+    extension_B_expected = ImmutableList.of("e");
 
     rootExtensionsDir = temporaryFolder.newFolder("extensions");
     rootHadoopDependenciesDir = temporaryFolder.newFolder("druid_hadoop_dependencies");
+    rootCommonDependenciesDir = temporaryFolder.newFolder("commonDeps");
 
     RepositorySystem realRepositorySystem = RealRepositorySystemUtil.newRepositorySystem();
     RepositorySystem spyMockRepositorySystem = spy(realRepositorySystem);
@@ -150,6 +158,12 @@ public String getHadoopDependenciesDir()
           {
             return rootHadoopDependenciesDir.getAbsolutePath();
           }
+
+          @Override
+          public String getCommonDependenciesDir()
+          {
+            return rootCommonDependenciesDir.getAbsolutePath();
+          }
         },
         HADOOP_EXCLUSIONS
     );
@@ -196,6 +210,20 @@ private DependencyResult mockDependencyResult(Artifact artifact)
     return result;
   }
 
+  private List<File> getExpectedJarFiles(List<String> expectedJarList, File baseDir, Artifact artifact)
+  {
+    return expectedJarList.stream()
+                          .map(name -> new File(
+                              StringUtils.format(
+                                  "%s/%s/%s",
+                                  baseDir,
+                                  artifact == null ? "" : artifact.getArtifactId(),
+                                  name + ".jar"
+                              )
+                          ))
+                          .collect(Collectors.toList());
+  }
+
   private List<File> getExpectedJarFiles(Artifact artifact)
   {
     final String artifactId = artifact.getArtifactId();
@@ -249,6 +277,26 @@ public void testPullDependencies_root_extension_dir_bad_state() throws IOExcepti
     pullDependencies.run();
   }
 
+  /**
+   * If --clean is not specified and common dependencies directory already exists, skip creating.
+   */
+  @Test()
+  public void testPullDependencies_root_common_dependencies_dir_exists()
+  {
+    pullDependencies.run();
+  }
+
+  /**
+   * A file exists on the root extension directory path, but it's not a directory, throw exception.
+   */
+  @Test(expected = RuntimeException.class)
+  public void testPullDependencies_root_common_dependencies_dir_bad_state() throws IOException
+  {
+    Assert.assertTrue(rootCommonDependenciesDir.delete());
+    Assert.assertTrue(rootCommonDependenciesDir.createNewFile());
+    pullDependencies.run();
+  }
+
   /**
    * If --clean is not specified and hadoop dependencies directory already exists, skip creating.
    */
@@ -281,11 +329,14 @@ public void testPullDependencies()
 
     final List<File> jarsUnderExtensionA = Arrays.asList(actualExtensions[0].listFiles());
     Collections.sort(jarsUnderExtensionA);
-    Assert.assertEquals(getExpectedJarFiles(extension_A), jarsUnderExtensionA);
+    Assert.assertEquals(getExpectedJarFiles(extension_A_expected, rootExtensionsDir, extension_A), jarsUnderExtensionA);
 
     final List<File> jarsUnderExtensionB = Arrays.asList(actualExtensions[1].listFiles());
     Collections.sort(jarsUnderExtensionB);
-    Assert.assertEquals(getExpectedJarFiles(extension_B), jarsUnderExtensionB);
+    Assert.assertEquals(getExpectedJarFiles(extension_B_expected, rootExtensionsDir, extension_B), jarsUnderExtensionB);
+
+    final List<File> jarsUnderCommon = Arrays.asList(rootCommonDependenciesDir.listFiles());
+    Assert.assertEquals(getExpectedJarFiles(commonDependencyList, rootCommonDependenciesDir, null), jarsUnderCommon);
 
     final File[] actualHadoopDependencies = rootHadoopDependenciesDir.listFiles();
     Arrays.sort(actualHadoopDependencies);