diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f9205759c..5c0fdc5b1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -35,6 +35,18 @@ jobs: run: | ./gradlew build -Dopensearch.version=1.2.0-SNAPSHOT + - name: Run k-NN Backwards Compatibility Tests + run: | + echo "Creating ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT ..." + mkdir -p ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT + echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT ..." + ls ./build/distributions/ + cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT + echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT ..." + ls ./src/test/resources/org/opensearch/knn/bwc/1.2.0.0-SNAPSHOT + echo "Running backwards compatibility tests ..." + ./gradlew bwcTestSuite -Dtests.security.manager=false + - name: Upload Coverage Report uses: codecov/codecov-action@v1 with: diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 5a4a201e0..5c70296f3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -12,6 +12,7 @@ - [Run Single-node Cluster Locally](#run-single-node-cluster-locally) - [Run Multi-node Cluster Locally](#run-multi-node-cluster-locally) - [Debugging](#debugging) + - [Backwards Compatibility Testing](#backwards-compatibility-testing) - [Submitting Changes](#submitting-changes) # Developer Guide @@ -177,6 +178,17 @@ Additionally, it is possible to attach one debugger to the cluster JVM and anoth ./gradlew :integTest -Dtest.debug=1 -Dcluster.debug=1 ``` +## Backwards Compatibility Testing + +The purpose of Backwards Compatibility Testing and different types of BWC tests are explained [here](https://github.com/opensearch-project/opensearch-plugins/blob/main/TESTING.md#backwards-compatibility-testing) + +Use these commands to run BWC tests for k-NN: + +1. Mixed cluster test: `./gradlew knnBwcCluster#mixedClusterTask -Dtests.security.manager=false` +2. Rolling upgrade tests: `./gradlew knnBwcCluster#rollingUpgradeClusterTask -Dtests.security.manager=false` +3. Full restart upgrade tests: `./gradlew knnBwcCluster#fullRestartClusterTask -Dtests.security.manager=false` +4. `./gradlew bwcTestSuite -Dtests.security.manager=false` is used to run all the above bwc tests together. + ## Submitting Changes See [CONTRIBUTING](CONTRIBUTING.md). \ No newline at end of file diff --git a/build.gradle b/build.gradle index 04d10a981..afbc9d75e 100644 --- a/build.gradle +++ b/build.gradle @@ -3,6 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ +import java.util.concurrent.Callable +import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask + buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "1.2.0-SNAPSHOT") @@ -170,6 +173,12 @@ integTest { systemProperty "user", System.getProperty("user") systemProperty "password", System.getProperty("password") + if (System.getProperty("tests.rest.bwcsuite") == null) { + filter { + excludeTestsMatching "org.opensearch.knn.bwc.*IT" + } + } + doFirst { // Tell the test JVM if the cluster JVM is running under a debugger so that tests can // use longer timeouts for requests. @@ -208,6 +217,153 @@ testClusters.integTest { systemProperty("java.library.path", "$rootDir/jni/release") } +// bwcVersion is the previous version of the k-NN plugin +// bwcFilePath contains the gradlew assemble binary files of k-NN plugins +String bwcVersion = "1.1.0" +String baseName = "knnBwcCluster" +String bwcFilePath = "src/test/resources/org/opensearch/knn/bwc/" + +// Creates two test clusters of previous version and loads k-NN plugin of bwcVersion +2.times { i -> + testClusters { + "${baseName}$i" { + testDistribution = "ARCHIVE" + versions = ["1.1.0", "1.2.0"] //Opensearch Cluster Versions + numberOfNodes = 3 + plugin(provider(new Callable() { + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return fileTree(bwcFilePath + bwcVersion).getSingleFile() + } + } + } + })) + setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}" + setting 'http.content_type.required', 'true' + systemProperty "java.library.path", "$rootDir/src/test/resources/org/opensearch/knn/bwc/lib:$rootDir/jni/release" + } + } +} + +// upgradeNodeAndPluginToNextVersion(plugins) upgrades plugin on the upgraded node with project.version binary file in bwcFilePath +// upgradeAllNodesAndPluginsToNextVersion(plugins) upgrades plugins on all the 3 nodes after upgrading the nodes +List> plugins = [ + provider(new Callable(){ + @Override + RegularFile call() throws Exception { + return new RegularFile() { + @Override + File getAsFile() { + return fileTree(bwcFilePath + project.version).getSingleFile() + } + } + } + }) +] + +// Creates 2 test clusters with 3 nodes of the old version. +2.times { i -> + task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}$i" + filter { + includeTestsMatching "org.opensearch.knn.bwc.*IT" + } + systemProperty 'tests.rest.bwcsuite', 'old_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'old' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}") + } +} + +// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version +// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. +// This is also used as a one third upgraded cluster for a rolling upgrade. +task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}0" + dependsOn "${baseName}#oldVersionClusterTask0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.knn.bwc.*IT" + } + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'first' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded. +// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. +// This is used for rolling upgrade. +task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#mixedClusterTask" + useCluster testClusters."${baseName}0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.knn.bwc.*IT" + } + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'second' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded. +// This results in a fully upgraded cluster. +// This is used for rolling upgrade. +task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#twoThirdsUpgradedClusterTask" + useCluster testClusters."${baseName}0" + doFirst { + testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.knn.bwc.*IT" + } + mustRunAfter "${baseName}#mixedClusterTask" + systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'third' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}") +} + +// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version +// at the same time resulting in a fully upgraded cluster. +task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) { + dependsOn "${baseName}#oldVersionClusterTask1" + useCluster testClusters."${baseName}1" + doFirst { + testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) + } + filter { + includeTestsMatching "org.opensearch.knn.bwc.*IT" + } + systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster' + systemProperty 'tests.rest.bwcsuite_round', 'fullrestart' + systemProperty 'tests.plugin_bwc_version', bwcVersion + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}") +} + +// A bwc test suite which runs all the bwc tasks combined. +task bwcTestSuite(type: StandaloneRestIntegTestTask) { + exclude '**/*Test*' + exclude '**/*IT*' + dependsOn tasks.named("${baseName}#mixedClusterTask") + dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") + dependsOn tasks.named("${baseName}#fullRestartClusterTask") +} + run { useCluster project.testClusters.integTest dependsOn buildJniLib diff --git a/src/test/java/org/opensearch/knn/ODFERestTestCase.java b/src/test/java/org/opensearch/knn/ODFERestTestCase.java index 17d99a0cb..1a2f10ab0 100644 --- a/src/test/java/org/opensearch/knn/ODFERestTestCase.java +++ b/src/test/java/org/opensearch/knn/ODFERestTestCase.java @@ -35,6 +35,8 @@ import org.opensearch.test.rest.OpenSearchRestTestCase; import org.junit.After; +import static org.opensearch.knn.TestUtils.KNN_BWC_PREFIX; + /** * ODFE integration test base class to support both security disabled and enabled ODFE cluster. */ @@ -138,8 +140,8 @@ protected void wipeAllODFEIndices() throws IOException { for (Map index : parserList) { String indexName = (String) index.get("index"); - if (indexName != null && !".opendistro_security".equals(indexName)) { - client().performRequest(new Request("DELETE", "/" + indexName)); + if (indexName != null && !".opendistro_security".equals(indexName) && !indexName.matches(KNN_BWC_PREFIX+"(.*)")) { + adminClient().performRequest(new Request("DELETE", "/" + indexName)); } } } diff --git a/src/test/java/org/opensearch/knn/TestUtils.java b/src/test/java/org/opensearch/knn/TestUtils.java index 04b8dbe22..cf6afe98d 100644 --- a/src/test/java/org/opensearch/knn/TestUtils.java +++ b/src/test/java/org/opensearch/knn/TestUtils.java @@ -25,6 +25,7 @@ import java.util.Map; public class TestUtils { + public static final String KNN_BWC_PREFIX = "knn-bwc-"; /** * Class to read in some test data from text files */ diff --git a/src/test/java/org/opensearch/knn/bwc/KNNBackwardsCompatibilityIT.java b/src/test/java/org/opensearch/knn/bwc/KNNBackwardsCompatibilityIT.java new file mode 100644 index 000000000..3f7af1dca --- /dev/null +++ b/src/test/java/org/opensearch/knn/bwc/KNNBackwardsCompatibilityIT.java @@ -0,0 +1,193 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.bwc; + +import java.util.*; +import java.util.stream.Collectors; +import org.apache.http.util.EntityUtils; +import org.opensearch.knn.KNNResult; +import org.opensearch.knn.index.KNNQueryBuilder; +import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.junit.Assert; +import org.opensearch.common.settings.Settings; +import org.opensearch.test.rest.OpenSearchRestTestCase; + +public class KNNBackwardsCompatibilityIT extends KNNRestTestCase { + private static final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.bwcsuite")); + private static final String CLUSTER_NAME = System.getProperty("tests.clustername"); + private final String testIndexName = "knn-bwc-test-index"; + private final String testFieldName = "knn-bwc-test-field"; + private final int dimensions = 2; + + @Override + protected final boolean preserveIndicesUponCompletion() { + return true; + } + + @Override + protected final boolean preserveReposUponCompletion() { + return true; + } + + @Override + protected boolean preserveTemplatesUponCompletion() { + return true; + } + + @Override + protected final Settings restClientSettings() { + return Settings + .builder() + .put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(OpenSearchRestTestCase.CLIENT_SOCKET_TIMEOUT, "90s") + .build(); + } + + private enum ClusterType { + OLD, + MIXED, + UPGRADED; + + public static ClusterType parse(String value) { + switch (value) { + case "old_cluster": + return OLD; + case "mixed_cluster": + return MIXED; + case "upgraded_cluster": + return UPGRADED; + default: + throw new AssertionError("unknown cluster type: " + value); + } + } + } + +// Use this prefix "knn-bwc-" while creating a test index to test BWC Tests. +// For example: testIndexName = "knn-bwc-test-index" + @SuppressWarnings("unchecked") + public void testBackwardsCompatibility() throws Exception { + String uri = getUri(); + Map> responseMap = (Map>) getAsMap(uri).get("nodes"); + for (Map response : responseMap.values()) { + List> plugins = (List>) response.get("plugins"); + Set pluginNames = plugins.stream().map(map -> map.get("name")).collect(Collectors.toSet()); + switch (CLUSTER_TYPE) { + case OLD: + Assert.assertTrue(pluginNames.contains("opensearch-knn")); + + Request waitForGreen = new Request("GET", "/_cluster/health"); + waitForGreen.addParameter("wait_for_nodes", "3"); + waitForGreen.addParameter("wait_for_status", "green"); + client().performRequest(waitForGreen); + + int graphCountBefore = getTotalGraphsInCache(); + + createKnnIndex(testIndexName, getKNNDefaultIndexSettings(), createKnnIndexMapping(testFieldName, dimensions)); + addKnnDoc(testIndexName, "1", testFieldName, new Float[]{6.0f, 6.0f}); + + knnWarmup(Collections.singletonList(testIndexName)); + + assertEquals(graphCountBefore + 1, getTotalGraphsInCache()); + + break; + case MIXED: + Assert.assertTrue(pluginNames.contains("opensearch-knn")); + + float[] queryVector = {10.0f, 10.0f}; + int k = 1; + + KNNQueryBuilder knnQueryBuilder = new KNNQueryBuilder(testFieldName, queryVector, k); + Response resp = searchKNNIndex(testIndexName, knnQueryBuilder,k); + List results = parseSearchResponse(EntityUtils.toString(resp.getEntity()), testFieldName); + + assertEquals(results.size(), k); + for(KNNResult result : results) { + String round = System.getProperty("tests.rest.bwcsuite_round"); + if(round.equals("first")) { + assertEquals("1", result.getDocId()); + int graphCountFirst = getTotalGraphsInCache(); + + deleteKnnDoc(testIndexName, "1"); + addKnnDoc(testIndexName, "2", testFieldName, new Float[]{50.0f, 50.0f}); + addKnnDoc(testIndexName, "4", testFieldName, new Float[]{55.0f, 55.0f}); + + knnWarmup(Collections.singletonList(testIndexName)); + assertEquals(graphCountFirst + 2, getTotalGraphsInCache()); + } + else if(round.equals("second")) { + assertEquals("2", result.getDocId()); + int graphCountSecond = getTotalGraphsInCache(); + + deleteKnnDoc(testIndexName, "2"); + + knnWarmup(Collections.singletonList(testIndexName)); + assertEquals(graphCountSecond, getTotalGraphsInCache()); + } + else { + assertEquals("4", result.getDocId()); + deleteKNNIndex(testIndexName); + } + } + + break; + case UPGRADED: + Assert.assertTrue(pluginNames.contains("opensearch-knn")); + + int graphCountUpgraded = getTotalGraphsInCache(); + + updateKnnDoc(testIndexName, "1", testFieldName, new Float[]{17.0f, 17.0f}); + + addKnnDoc(testIndexName, "3", testFieldName, new Float[]{20.0f, 20.0f}); + + knnWarmup(Collections.singletonList(testIndexName)); + assertEquals(graphCountUpgraded+3, getTotalGraphsInCache()); + + forceMergeKnnIndex(testIndexName); + + float[] queryVector1 = {15.0f, 15.0f}; + int k1 = 1; + + KNNQueryBuilder knnQueryBuilder1 = new KNNQueryBuilder(testFieldName, queryVector1, k1); + Response resp1 = searchKNNIndex(testIndexName, knnQueryBuilder1,k1); + List results1 = parseSearchResponse(EntityUtils.toString(resp1.getEntity()), testFieldName); + + assertEquals(results1.size(), k1); + for(KNNResult result : results1) + assertEquals("1", result.getDocId()); + + deleteKNNIndex(testIndexName); + + break; + } + break; + } + } + + private String getUri() { + switch (CLUSTER_TYPE) { + case OLD: + return "_nodes/" + CLUSTER_NAME + "-0/plugins"; + case MIXED: + String round = System.getProperty("tests.rest.bwcsuite_round"); + if (round.equals("second")) { + return "_nodes/" + CLUSTER_NAME + "-1/plugins"; + } else if (round.equals("third")) { + return "_nodes/" + CLUSTER_NAME + "-2/plugins"; + } else { + return "_nodes/" + CLUSTER_NAME + "-0/plugins"; + } + case UPGRADED: + return "_nodes/plugins"; + default: + throw new AssertionError("unknown cluster type: " + CLUSTER_TYPE); + } + } +} diff --git a/src/test/resources/org/opensearch/knn/bwc/1.0.0/opensearch-knn-1.0.0.0.zip b/src/test/resources/org/opensearch/knn/bwc/1.0.0/opensearch-knn-1.0.0.0.zip new file mode 100644 index 000000000..7bdcf6faf Binary files /dev/null and b/src/test/resources/org/opensearch/knn/bwc/1.0.0/opensearch-knn-1.0.0.0.zip differ diff --git a/src/test/resources/org/opensearch/knn/bwc/1.1.0/opensearch-knn-1.1.0.0.zip b/src/test/resources/org/opensearch/knn/bwc/1.1.0/opensearch-knn-1.1.0.0.zip new file mode 100644 index 000000000..c64e567ba Binary files /dev/null and b/src/test/resources/org/opensearch/knn/bwc/1.1.0/opensearch-knn-1.1.0.0.zip differ diff --git a/src/test/resources/org/opensearch/knn/bwc/lib/libKNNIndexV2_0_11.so b/src/test/resources/org/opensearch/knn/bwc/lib/libKNNIndexV2_0_11.so new file mode 100644 index 000000000..6db6a9da2 Binary files /dev/null and b/src/test/resources/org/opensearch/knn/bwc/lib/libKNNIndexV2_0_11.so differ