Adding Backward Compatibility Tests for k-NN

Signed-off-by: Naveen Tatikonda <navtat@amazon.com>
opensearch-project · Nov 23, 2021 · 24add37 · 24add37
1 parent 1ff98c8
commit 24add37
Show file tree

Hide file tree

Showing 9 changed files with 378 additions and 2 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -35,6 +35,18 @@ jobs:
         run: |
           ./gradlew build -Dopensearch.version=1.2.0-SNAPSHOT
 
+      - name: Run k-NN Backwards Compatibility Tests
+        run: |
+          echo "Creating ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT ..."
+          mkdir -p ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT
+          echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT ..."
+          ls ./build/distributions/
+          cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT
+          echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT ..."
+          ls ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT
+          echo "Running backwards compatibility tests ..."
+          ./gradlew bwcTestSuite -Dtests.security.manager=false
+
       - name: Upload Coverage Report
         uses: codecov/codecov-action@v1
         with:

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -12,6 +12,7 @@
     - [Run Single-node Cluster Locally](#run-single-node-cluster-locally)
     - [Run Multi-node Cluster Locally](#run-multi-node-cluster-locally)
   - [Debugging](#debugging)
+  - [Backwards Compatibility Testing](#backwards-compatibility-testing)
   - [Submitting Changes](#submitting-changes)
 
 # Developer Guide
@@ -177,6 +178,17 @@ Additionally, it is possible to attach one debugger to the cluster JVM and anoth
 ./gradlew :integTest -Dtest.debug=1 -Dcluster.debug=1
 ```
 
+## Backwards Compatibility Testing
+
+The purpose of Backwards Compatibility Testing and different types of BWC tests are explained [here](https://github.com/opensearch-project/opensearch-plugins/blob/main/TESTING.md#backwards-compatibility-testing)
+
+Use these commands to run BWC tests for k-NN:
+
+1. Mixed cluster test: `./gradlew knnBwcCluster#mixedClusterTask -Dtests.security.manager=false`
+2. Rolling upgrade tests: `./gradlew knnBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false`
+3. Full restart upgrade tests: `./gradlew knnBwcCluster#fullRestartClusterTask -Dtests.security.manager=false`
+4. `./gradlew bwcTestSuite -Dtests.security.manager=false` is used to run all the above bwc tests together.
+
 ## Submitting Changes
 
 See [CONTRIBUTING](CONTRIBUTING.md).
diff --git a/build.gradle b/build.gradle
@@ -3,6 +3,9 @@
  *  SPDX-License-Identifier: Apache-2.0
  */
 
+import java.util.concurrent.Callable
+import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
+
 buildscript {
     ext {
         opensearch_version = System.getProperty("opensearch.version", "1.2.0-SNAPSHOT")
@@ -170,6 +173,12 @@ integTest {
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
 
+    if (System.getProperty("tests.rest.bwcsuite") == null) {
+        filter {
+            excludeTestsMatching "org.opensearch.knn.bwc.*IT"
+        }
+    }
+
     doFirst {
         // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
         // use longer timeouts for requests.
@@ -208,6 +217,153 @@ testClusters.integTest {
     systemProperty("java.library.path", "$rootDir/jni/release")
 }
 
+// bwcVersion is the previous version of the k-NN plugin
+// bwcFilePath contains the gradlew assemble binary files of k-NN plugins
+String bwcVersion = "1.1.0"
+String baseName = "knnBwcCluster"
+String bwcFilePath = "src/test/resources/org/opensearch/knn/bwc/"
+
+// Creates two test clusters of previous version and loads k-NN plugin of bwcVersion
+2.times { i ->
+    testClusters {
+        "${baseName}$i" {
+            testDistribution = "ARCHIVE"
+            versions = ["1.1.0", "1.2.0"]   //Opensearch Cluster Versions
+            numberOfNodes = 3
+            plugin(provider(new Callable<RegularFile>() {
+                @Override
+                RegularFile call() throws Exception {
+                    return new RegularFile() {
+                        @Override
+                        File getAsFile() {
+                            return fileTree(bwcFilePath + bwcVersion).getSingleFile()
+                        }
+                    }
+                }
+            }))
+            setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
+            setting 'http.content_type.required', 'true'
+            systemProperty "java.library.path", "$rootDir/src/test/resources/org/opensearch/knn/bwc/lib:$rootDir/jni/release"
+        }
+    }
+}
+
+// upgradeNodeAndPluginToNextVersion(plugins) upgrades plugin on the upgraded node with project.version binary file in bwcFilePath
+// upgradeAllNodesAndPluginsToNextVersion(plugins) upgrades plugins on all the 3 nodes after upgrading the nodes
+List<Provider<RegularFile>> plugins = [
+        provider(new Callable<RegularFile>(){
+            @Override
+            RegularFile call() throws Exception {
+                return new RegularFile() {
+                    @Override
+                    File getAsFile() {
+                        return fileTree(bwcFilePath + project.version).getSingleFile()
+                    }
+                }
+            }
+        })
+]
+
+// Creates 2 test clusters with 3 nodes of the old version.
+2.times { i ->
+    task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
+        useCluster testClusters."${baseName}$i"
+        filter {
+            includeTestsMatching "org.opensearch.knn.bwc.*IT"
+        }
+        systemProperty 'tests.rest.bwcsuite', 'old_cluster'
+        systemProperty 'tests.rest.bwcsuite_round', 'old'
+        systemProperty 'tests.plugin_bwc_version', bwcVersion
+        nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
+        nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
+    }
+}
+
+// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
+// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
+// This is also used as a one third upgraded cluster for a rolling upgrade.
+task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
+    useCluster testClusters."${baseName}0"
+    dependsOn "${baseName}#oldVersionClusterTask0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.knn.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'first'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
+// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
+// This is used for rolling upgrade.
+task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#mixedClusterTask"
+    useCluster testClusters."${baseName}0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.knn.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'second'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
+// This results in a fully upgraded cluster.
+// This is used for rolling upgrade.
+task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
+    useCluster testClusters."${baseName}0"
+    doFirst {
+        testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.knn.bwc.*IT"
+    }
+    mustRunAfter "${baseName}#mixedClusterTask"
+    systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'third'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
+}
+
+// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
+// at the same time resulting in a fully upgraded cluster.
+task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
+    dependsOn "${baseName}#oldVersionClusterTask1"
+    useCluster testClusters."${baseName}1"
+    doFirst {
+        testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
+    }
+    filter {
+        includeTestsMatching "org.opensearch.knn.bwc.*IT"
+    }
+    systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
+    systemProperty 'tests.rest.bwcsuite_round', 'fullrestart'
+    systemProperty 'tests.plugin_bwc_version', bwcVersion
+    nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
+    nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
+}
+
+// A bwc test suite which runs all the bwc tasks combined.
+task bwcTestSuite(type: StandaloneRestIntegTestTask) {
+    exclude '**/*Test*'
+    exclude '**/*IT*'
+    dependsOn tasks.named("${baseName}#mixedClusterTask")
+    dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
+    dependsOn tasks.named("${baseName}#fullRestartClusterTask")
+}
+
 run {
     useCluster project.testClusters.integTest
     dependsOn buildJniLib

diff --git a/src/test/java/org/opensearch/knn/ODFERestTestCase.java b/src/test/java/org/opensearch/knn/ODFERestTestCase.java
@@ -35,6 +35,8 @@
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 import org.junit.After;
 
+import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX;
+
 /**
  * ODFE integration test base class to support both security disabled and enabled ODFE cluster.
  */
@@ -138,8 +140,8 @@ protected void wipeAllODFEIndices() throws IOException {
 
             for (Map<String, Object> index : parserList) {
                 String indexName = (String) index.get("index");
-                if (indexName != null && !".opendistro_security".equals(indexName)) {
-                    client().performRequest(new Request("DELETE", "/" + indexName));
+                if (indexName != null && !".opendistro_security".equals(indexName) && !indexName.matches(KNN_BWC_PREFIX+"(.*)")) {
+                    adminClient().performRequest(new Request("DELETE", "/" + indexName));
                 }
             }
         }

diff --git a/src/test/java/org/opensearch/knn/TestUtils.java b/src/test/java/org/opensearch/knn/TestUtils.java
@@ -25,6 +25,7 @@
 import java.util.Map;
 
 public class TestUtils {
+    public static final String KNN_BWC_PREFIX = "knn-bwc-";
     /**
      * Class to read in some test data from text files
      */