Client-side encrypted snapshot repository (feature flag) (#66773)

The client-side encrypted repository is a new type of snapshot repository that
internally delegates to the regular variants of snapshot repositories (of types
Azure, S3, GCS, FS, and maybe others but not yet tested). After the encrypted
repository is set up, it is transparent to the snapshot and restore APIs (i.e. all
snapshots stored in the encrypted repository are encrypted, no other parameters
required).
The encrypted repository is protected by a password stored on every node's
keystore (which must be the same across the nodes).
The password is used to generate a key encrytion key (KEK), using the PBKDF2
function, which is used to encrypt (using the AES Wrap algorithm) other
symmetric keys (referred to as DEK - data encryption keys), which themselves
are generated randomly, and which are ultimately used to encrypt the snapshot
blobs.

For example, here is how to set up an encrypted  FS repository:
------
 1) make sure that the cluster runs under at least a "platinum" license
(simplest test configuration is to put `xpack.license.self_generated.type: "trial"`
in the elasticsearch.yml file)
 2) identical to the un-encrypted FS repository, specify the mount point of the
shared FS in the elasticsearch.yml conf file (on all the cluster nodes),
e.g. `path.repo: ["/tmp/repo"]`
 3) store the repository password inside the elasticsearch.keystore, *on every cluster node*.
In order to support changing password on existing repository (implemented in a follow-up),
the password itself must be names, e.g. for the "test_enc_key" repository password name:
`./bin/elasticsearch-keystore add repository.encrypted.test_enc_pass.password`
*type in the password*
4) start up the cluster and create the new encrypted FS repository, named "test_enc", by calling:
`
curl -X PUT "localhost:9200/_snapshot/test_enc?pretty" -H 'Content-Type: application/json' -d'
{
  "type": "encrypted",
  "settings": {
    "location": "/tmp/repo/enc",
    "delegate_type": "fs",
    "password_name": "test_enc_pass"
  }
}
'
`
5) the snapshot and restore APIs work unmodified when they refer to this new repository, e.g.
` curl -X PUT "localhost:9200/_snapshot/test_enc/snapshot_1?wait_for_completion=true"`


Related: #49896 #41910 #50846 #48221 #65768

plugins/repository-azure/build.gradle

@@ -374,3 +374,18 @@ task azureThirdPartyTest(type: Test) {
   }
 }
 tasks.named("check").configure { dependsOn("azureThirdPartyTest") }
+
+// test jar is exported by the integTestArtifacts configuration to be used in the encrypted Azure repository test
+configurations {
+  internalClusterTestArtifacts.extendsFrom internalClusterTestImplementation
+  internalClusterTestArtifacts.extendsFrom internalClusterTestRuntime
+}
+
+def internalClusterTestJar = tasks.register("internalClusterTestJar", Jar) {
+  appendix 'internalClusterTest'
+  from sourceSets.internalClusterTest.output
+}
+
+artifacts {
+  internalClusterTestArtifacts internalClusterTestJar
+}

plugins/repository-azure/src/internalClusterTest/java/org/elasticsearch/repositories/azure/AzureBlobStoreRepositoryTests.java

@@ -59,12 +59,15 @@ protected String repositoryType() {
     }
 
     @Override
-    protected Settings repositorySettings() {
-        return Settings.builder()
-            .put(super.repositorySettings())
-            .put(AzureRepository.Repository.CONTAINER_SETTING.getKey(), "container")
-            .put(AzureStorageSettings.ACCOUNT_SETTING.getKey(), "test")
-            .build();
+    protected Settings repositorySettings(String repoName) {
+        Settings.Builder settingsBuilder = Settings.builder()
+                .put(super.repositorySettings(repoName))
+                .put(AzureRepository.Repository.CONTAINER_SETTING.getKey(), "container")
+                .put(AzureStorageSettings.ACCOUNT_SETTING.getKey(), "test");
+        if (randomBoolean()) {
+            settingsBuilder.put(AzureRepository.Repository.BASE_PATH_SETTING.getKey(), randomFrom("test", "test/1"));
+        }
+        return settingsBuilder.build();
     }
 
     @Override

plugins/repository-gcs/build.gradle

@@ -334,3 +334,20 @@ def gcsThirdPartyTest = tasks.register("gcsThirdPartyTest", Test) {
 tasks.named("check").configure {
   dependsOn(largeBlobYamlRestTest, gcsThirdPartyTest)
 }
+
+// test jar is exported by the integTestArtifacts configuration to be used in the encrypted GCS repository test
+configurations {
+  internalClusterTestArtifacts.extendsFrom internalClusterTestImplementation
+  internalClusterTestArtifacts.extendsFrom internalClusterTestRuntime
+}
+
+def internalClusterTestJar = tasks.register("internalClusterTestJar", Jar) {
+  appendix 'internalClusterTest'
+  from sourceSets.internalClusterTest.output
+  // for the repositories.gcs.TestUtils class
+  from sourceSets.test.output
+}
+
+artifacts {
+  internalClusterTestArtifacts internalClusterTestJar
+}

plugins/repository-gcs/src/internalClusterTest/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -67,6 +67,7 @@
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.TOKEN_URI_SETTING;
+import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.BASE_PATH;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.BUCKET;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageRepository.CLIENT_NAME;
 
@@ -79,12 +80,15 @@ protected String repositoryType() {
     }
 
     @Override
-    protected Settings repositorySettings() {
-        return Settings.builder()
-            .put(super.repositorySettings())
-            .put(BUCKET.getKey(), "bucket")
-            .put(CLIENT_NAME.getKey(), "test")
-            .build();
+    protected Settings repositorySettings(String repoName) {
+        Settings.Builder settingsBuilder = Settings.builder()
+                .put(super.repositorySettings(repoName))
+                .put(BUCKET.getKey(), "bucket")
+                .put(CLIENT_NAME.getKey(), "test");
+        if (randomBoolean()) {
+            settingsBuilder.put(BASE_PATH.getKey(), randomFrom("test", "test/1"));
+        }
+        return settingsBuilder.build();
     }
 
     @Override
@@ -120,7 +124,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
     }
 
     public void testDeleteSingleItem() {
-        final String repoName = createRepository(randomName());
+        final String repoName = createRepository(randomRepositoryName());
         final RepositoriesService repositoriesService = internalCluster().getMasterNodeInstance(RepositoriesService.class);
         final BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(repoName);
         PlainActionFuture.get(f -> repository.threadPool().generic().execute(ActionRunnable.run(f, () ->

plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreRepositoryTests.java

@@ -38,7 +38,7 @@ protected String repositoryType() {
     }
 
     @Override
-    protected Settings repositorySettings() {
+    protected Settings repositorySettings(String repoName) {
         return Settings.builder()
             .put("uri", "hdfs:///")
             .put("conf.fs.AbstractFileSystem.hdfs.impl", TestingFs.class.getName())
@@ -47,6 +47,12 @@ protected Settings repositorySettings() {
             .put("compress", randomBoolean()).build();
     }
 
+    @Override
+    public void testSnapshotAndRestore() throws Exception {
+        // the HDFS mockup doesn't preserve the repository contents after removing the repository
+        testSnapshotAndRestore(false);
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singletonList(HdfsPlugin.class);

plugins/repository-s3/build.gradle

@@ -312,3 +312,20 @@ tasks.named("thirdPartyAudit").configure {
           'javax.activation.DataHandler'
   )
 }
+
+// test jar is exported by the integTestArtifacts configuration to be used in the encrypted S3 repository test
+configurations {
+  internalClusterTestArtifacts.extendsFrom internalClusterTestImplementation
+  internalClusterTestArtifacts.extendsFrom internalClusterTestRuntime
+}
+
+def internalClusterTestJar = tasks.register("internalClusterTestJar", Jar) {
+  appendix 'internalClusterTest'
+  from sourceSets.internalClusterTest.output
+  // for the plugin-security.policy resource
+  from sourceSets.test.output
+}
+
+artifacts {
+  internalClusterTestArtifacts internalClusterTestJar
+}

plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -94,14 +94,17 @@ protected String repositoryType() {
     }
 
     @Override
-    protected Settings repositorySettings() {
-        return Settings.builder()
-            .put(super.repositorySettings())
+    protected Settings repositorySettings(String repoName) {
+        Settings.Builder settingsBuilder = Settings.builder()
+            .put(super.repositorySettings(repoName))
             .put(S3Repository.BUCKET_SETTING.getKey(), "bucket")
             .put(S3Repository.CLIENT_NAME.getKey(), "test")
             // Don't cache repository data because some tests manually modify the repository data
-            .put(BlobStoreRepository.CACHE_REPOSITORY_DATA.getKey(), false)
-            .build();
+            .put(BlobStoreRepository.CACHE_REPOSITORY_DATA.getKey(), false);
+        if (randomBoolean()) {
+            settingsBuilder.put(S3Repository.BASE_PATH_SETTING.getKey(), randomFrom("test", "test/1"));
+        }
+        return settingsBuilder.build();
     }
 
     @Override
@@ -145,8 +148,9 @@ protected Settings nodeSettings(int nodeOrdinal) {
     }
 
     public void testEnforcedCooldownPeriod() throws IOException {
-        final String repoName = createRepository(randomName(), Settings.builder().put(repositorySettings())
-            .put(S3Repository.COOLDOWN_PERIOD.getKey(), TEST_COOLDOWN_PERIOD).build());
+        final String repoName = randomRepositoryName();
+        createRepository(repoName, Settings.builder().put(repositorySettings(repoName))
+                .put(S3Repository.COOLDOWN_PERIOD.getKey(), TEST_COOLDOWN_PERIOD).build(), true);
 
         final SnapshotId fakeOldSnapshot = client().admin().cluster().prepareCreateSnapshot(repoName, "snapshot-old")
             .setWaitForCompletion(true).setIndices().get().getSnapshotInfo().snapshotId();

server/src/internalClusterTest/java/org/elasticsearch/repositories/fs/FsBlobStoreRepositoryIntegTests.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.repositories.fs;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.repositories.blobstore.ESFsBasedRepositoryIntegTestCase;
+
+public class FsBlobStoreRepositoryIntegTests extends ESFsBasedRepositoryIntegTestCase {
+
+    @Override
+    protected Settings repositorySettings(String repositoryName) {
+        final Settings.Builder settings = Settings.builder()
+                .put("compress", randomBoolean())
+                .put("location", randomRepoPath());
+        if (randomBoolean()) {
+            long size = 1 << randomInt(10);
+            settings.put("chunk_size", new ByteSizeValue(size, ByteSizeUnit.KB));
+        }
+        return settings.build();
+    }
+}

server/src/main/java/org/elasticsearch/common/blobstore/BlobPath.java

@@ -25,6 +25,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * The list of paths where a blob can reside.  The contents of the paths are dependent upon the implementation of {@link BlobContainer}.
@@ -90,4 +91,17 @@ public String toString() {
         }
         return sb.toString();
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BlobPath other = (BlobPath) o;
+        return paths.equals(other.paths);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(paths);
+    }
 }

test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.repositories.RepositoryData;
+import org.elasticsearch.repositories.RepositoryMissingException;
 import org.elasticsearch.snapshots.SnapshotMissingException;
 import org.elasticsearch.snapshots.SnapshotRestoreException;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -78,17 +79,19 @@ public static RepositoryData getRepositoryData(Repository repository) {
 
     protected abstract String repositoryType();
 
-    protected Settings repositorySettings() {
+    protected Settings repositorySettings(String repoName) {
         return Settings.builder().put("compress", randomBoolean()).build();
     }
 
     protected final String createRepository(final String name) {
-        return createRepository(name, repositorySettings());
+        return createRepository(name, true);
     }
 
-    protected final String createRepository(final String name, final Settings settings) {
-        final boolean verify = randomBoolean();
+    protected final String createRepository(final String name, final boolean verify) {
+        return createRepository(name, repositorySettings(name), verify);
+    }
 
+    protected final String createRepository(final String name, final Settings settings, final boolean verify) {
         logger.info("-->  creating repository [name: {}, verify: {}, settings: {}]", name, verify, settings);
         assertAcked(client().admin().cluster().preparePutRepository(name)
             .setType(repositoryType())
@@ -98,14 +101,23 @@ protected final String createRepository(final String name, final Settings settin
         internalCluster().getDataOrMasterNodeInstances(RepositoriesService.class).forEach(repositories -> {
             assertThat(repositories.repository(name), notNullValue());
             assertThat(repositories.repository(name), instanceOf(BlobStoreRepository.class));
-            assertThat(repositories.repository(name).isReadOnly(), is(false));
+            assertThat(repositories.repository(name).isReadOnly(), is(settings.getAsBoolean("readonly", false)));
             BlobStore blobStore = ((BlobStoreRepository) repositories.repository(name)).getBlobStore();
             assertThat("blob store has to be lazy initialized", blobStore, verify ? is(notNullValue()) : is(nullValue()));
         });
 
         return name;
     }
 
+    protected final void deleteRepository(final String name) {
+        logger.debug("-->  deleting repository [name: {}]", name);
+        assertAcked(client().admin().cluster().prepareDeleteRepository(name));
+        internalCluster().getDataOrMasterNodeInstances(RepositoriesService.class).forEach(repositories -> {
+            RepositoryMissingException e = expectThrows(RepositoryMissingException.class, () -> repositories.repository(name));
+            assertThat(e.repository(), equalTo(name));
+        });
+    }
+
     public void testReadNonExistingPath() throws IOException {
         try (BlobStore store = newBlobStore()) {
             final BlobContainer container = store.blobContainer(new BlobPath());
@@ -176,7 +188,7 @@ public void testList() throws IOException {
                 BlobMetadata blobMetadata = blobs.get(generated.getKey());
                 assertThat(generated.getKey(), blobMetadata, CoreMatchers.notNullValue());
                 assertThat(blobMetadata.name(), CoreMatchers.equalTo(generated.getKey()));
-                assertThat(blobMetadata.length(), CoreMatchers.equalTo(generated.getValue()));
+                assertThat(blobMetadata.length(), CoreMatchers.equalTo(blobLengthFromContentLength(generated.getValue())));
             }
 
             assertThat(container.listBlobsByPrefix("foo-").size(), CoreMatchers.equalTo(numberOfFooBlobs));
@@ -259,15 +271,25 @@ protected static void writeBlob(BlobContainer container, String blobName, BytesA
     }
 
     protected BlobStore newBlobStore() {
-        final String repository = createRepository(randomName());
+        final String repository = createRepository(randomRepositoryName());
+        return newBlobStore(repository);
+    }
+
+    protected BlobStore newBlobStore(String repository) {
         final BlobStoreRepository blobStoreRepository =
             (BlobStoreRepository) internalCluster().getMasterNodeInstance(RepositoriesService.class).repository(repository);
         return PlainActionFuture.get(
             f -> blobStoreRepository.threadPool().generic().execute(ActionRunnable.supply(f, blobStoreRepository::blobStore)));
     }
 
     public void testSnapshotAndRestore() throws Exception {
-        final String repoName = createRepository(randomName());
+        testSnapshotAndRestore(randomBoolean());
+    }
+
+    protected void testSnapshotAndRestore(boolean recreateRepositoryBeforeRestore) throws Exception {
+        final String repoName = randomRepositoryName();
+        final Settings repoSettings = repositorySettings(repoName);
+        createRepository(repoName, repoSettings, randomBoolean());
         int indexCount = randomIntBetween(1, 5);
         int[] docCounts = new int[indexCount];
         String[] indexNames = generateRandomNames(indexCount);
@@ -315,6 +337,11 @@ public void testSnapshotAndRestore() throws Exception {
             assertAcked(client().admin().indices().prepareClose(closeIndices.toArray(new String[closeIndices.size()])));
         }
 
+        if (recreateRepositoryBeforeRestore) {
+            deleteRepository(repoName);
+            createRepository(repoName, repoSettings, randomBoolean());
+        }
+
         logger.info("--> restore all indices from the snapshot");
         assertSuccessfulRestore(client().admin().cluster().prepareRestoreSnapshot(repoName, snapshotName).setWaitForCompletion(true));
 
@@ -339,7 +366,7 @@ public void testSnapshotAndRestore() throws Exception {
     }
 
     public void testMultipleSnapshotAndRollback() throws Exception {
-        final String repoName = createRepository(randomName());
+        final String repoName = createRepository(randomRepositoryName());
         int iterationCount = randomIntBetween(2, 5);
         int[] docCounts = new int[iterationCount];
         String indexName = randomName();
@@ -394,7 +421,7 @@ public void testMultipleSnapshotAndRollback() throws Exception {
     }
 
     public void testIndicesDeletedFromRepository() throws Exception {
-        final String repoName = createRepository("test-repo");
+        final String repoName = createRepository(randomRepositoryName());
         Client client = client();
         createIndex("test-idx-1", "test-idx-2", "test-idx-3");
         ensureGreen();
@@ -491,7 +518,15 @@ private static void assertSuccessfulRestore(RestoreSnapshotResponse response) {
         assertThat(response.getRestoreInfo().successfulShards(), equalTo(response.getRestoreInfo().totalShards()));
     }
 
-    protected static String randomName() {
+    protected String randomName() {
         return randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);
     }
+
+    protected String randomRepositoryName() {
+        return randomName();
+    }
+
+    protected long blobLengthFromContentLength(long contentLength) {
+        return contentLength;
+    }
 }