[Backport 2.4] Add CI bundle pattern to distribution download (#5588)

* Backport Add CI bundle pattern to distribution download Signed-off-by: Zelin Hao <zelinhao@amazon.com> * Fix tests Signed-off-by: Zelin Hao <zelinhao@amazon.com> Signed-off-by: Zelin Hao <zelinhao@amazon.com>
opensearch-project · Dec 20, 2022 · 366ebc1 · 366ebc1
1 parent 6c19239
commit 366ebc1
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 12 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -32,6 +32,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add resource usage trackers for in-flight cancellation of SearchShardTask ([#4805](https://github.com/opensearch-project/OpenSearch/pull/4805))
 - Add search backpressure stats API ([#4932](https://github.com/opensearch-project/OpenSearch/pull/4932))
 - Add feature to ignore indexes starting with dot during shard limit validation.([#4695](https://github.com/opensearch-project/OpenSearch/pull/4695))
+- Add CI bundle pattern to distribution download ([#5348](https://github.com/opensearch-project/OpenSearch/pull/5348))
 
 ### Dependencies
 - Bump `com.diffplug.spotless` from 6.9.1 to 6.10.0

diff --git a/TESTING.md b/TESTING.md
@@ -383,6 +383,10 @@ Use -Dtest.class and -Dtests.method to run a specific bwcTest test. For example
      -Dtests.class=org.opensearch.upgrades.RecoveryIT \
      -Dtests.method=testHistoryUUIDIsGenerated
 
+Use `-PcustomDistributionDownloadType=bundle` to run the bwcTest against the test cluster with latest CI distribution bundle set up for the specified version; this property is default to min and exclusive choices between `bundle` and `min`:
+
+    ./gradlew bwcTest -PcustomDistributionDownloadType=bundle
+
 When running `./gradlew check`, minimal bwc checks are also run against compatible versions that are not yet released.
 
 ## BWC Testing against a specific remote/branch

diff --git a/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java
@@ -80,6 +80,8 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
     private static final String SNAPSHOT_PATTERN_LAYOUT =
         "/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier])-latest.[ext]";
+    private static final String BUNDLE_PATTERN_LAYOUT =
+        "/ci/dbc/distribution-build-opensearch/[revision]/latest/linux/x64/tar/dist/opensearch/[module]-[revision](-[classifier]).[ext]";
 
     private NamedDomainObjectContainer<OpenSearchDistribution> distributionsContainer;
     private NamedDomainObjectContainer<DistributionResolution> distributionsResolutionStrategiesContainer;
@@ -196,24 +198,46 @@ private static void setupDownloadServiceRepo(Project project) {
             return;
         }
         Object customDistributionUrl = project.findProperty("customDistributionUrl");
+        Object customDistributionDownloadType = project.findProperty("customDistributionDownloadType");
+        // distributionDownloadType is default min if is not specified; download the distribution from CI if is bundle
+        String distributionDownloadType = customDistributionDownloadType != null
+            && customDistributionDownloadType.toString().equals("bundle") ? "bundle" : "min";
+
+        addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
+        addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
+
         // checks if custom Distribution Url has been passed by user from plugins
         if (customDistributionUrl != null) {
             addIvyRepo(project, DOWNLOAD_REPO_NAME, customDistributionUrl.toString(), FAKE_IVY_GROUP, "");
             addIvyRepo(project, SNAPSHOT_REPO_NAME, customDistributionUrl.toString(), FAKE_SNAPSHOT_IVY_GROUP, "");
-        } else {
-            addIvyRepo(
-                project,
-                DOWNLOAD_REPO_NAME,
-                "https://artifacts.opensearch.org",
-                FAKE_IVY_GROUP,
-                "/releases" + RELEASE_PATTERN_LAYOUT,
-                "/release-candidates" + RELEASE_PATTERN_LAYOUT
-            );
-            addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
+            return;
+        }
+        switch (distributionDownloadType) {
+            case "bundle":
+                addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://ci.opensearch.org", FAKE_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
+                addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://ci.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
+                break;
+            case "min":
+                addIvyRepo(
+                    project,
+                    DOWNLOAD_REPO_NAME,
+                    "https://artifacts.opensearch.org",
+                    FAKE_IVY_GROUP,
+                    "/releases" + RELEASE_PATTERN_LAYOUT,
+                    "/release-candidates" + RELEASE_PATTERN_LAYOUT
+                );
+                addIvyRepo(
+                    project,
+                    SNAPSHOT_REPO_NAME,
+                    "https://artifacts.opensearch.org",
+                    FAKE_SNAPSHOT_IVY_GROUP,
+                    SNAPSHOT_PATTERN_LAYOUT
+                );
+                break;
+            default:
+                throw new IllegalArgumentException("Unsupported property argument: " + distributionDownloadType);
         }
 
-        addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
-        addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
     }
 
     /**