-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Backport 2.x] [Remote Store] Backport remote segment store changes (#…
…4380) * [Remote Store] Upload segments to remote store post refresh (#3460) * Add RemoteDirectory interface to copy segment files to/from remote store Signed-off-by: Sachin Kale <kalsac@amazon.com> Co-authored-by: Sachin Kale <kalsac@amazon.com> * Add index level setting for remote store Signed-off-by: Sachin Kale <kalsac@amazon.com> Co-authored-by: Sachin Kale <kalsac@amazon.com> * Add RemoteDirectoryFactory and use RemoteDirectory instance in RefreshListener Co-authored-by: Sachin Kale <kalsac@amazon.com> Signed-off-by: Sachin Kale <kalsac@amazon.com> * Upload segment to remote store post refresh Signed-off-by: Sachin Kale <kalsac@amazon.com> Co-authored-by: Sachin Kale <kalsac@amazon.com> Signed-off-by: Sachin Kale <kalsac@amazon.com> * [Remote Store] Inject remote store in IndexShard instead of RemoteStoreRefreshListener (#3703) * Inject remote store in IndexShard instead of RemoteStoreRefreshListener Signed-off-by: Sachin Kale <kalsac@amazon.com> * Pass supplier of RepositoriesService to RemoteDirectoryFactory Signed-off-by: Sachin Kale <kalsac@amazon.com> * Create isRemoteStoreEnabled function for IndexShard Signed-off-by: Sachin Kale <kalsac@amazon.com> * Explicitly close remoteStore on indexShard close Signed-off-by: Sachin Kale <kalsac@amazon.com> * Change RemoteDirectory.close to a no-op Signed-off-by: Sachin Kale <kalsac@amazon.com> Co-authored-by: Sachin Kale <kalsac@amazon.com> * [Remote Store] Add remote store restore API implementation (#3642) * Add remote restore API implementation Signed-off-by: Sachin Kale <kalsac@amazon.com> * [Remote Store] Add support to add nested settings for remote store (#4060) * Add support to add nested settings for remote store Signed-off-by: Sachin Kale <kalsac@amazon.com> * [Remote Store] Add rest endpoint for remote store restore (#3576) * Add rest endpoint for remote store restore Signed-off-by: Sachin Kale <kalsac@amazon.com> * [Remote Store] Add validator that forces segment replication type before enabling remote store (#4175) * Add validator that forces segment replication type before enabling remote store Signed-off-by: Sachin Kale <kalsac@amazon.com> * [Remote Store] Change remote_store setting validation message to make it more clear (#4199) * Change remote_store setting validation message to make it more clear Signed-off-by: Sachin Kale <kalsac@amazon.com> * [Remote Store] Add RemoteSegmentStoreDirectory to interact with remote segment store (#4020) * Add RemoteSegmentStoreDirectory to interact with remote segment store Signed-off-by: Sachin Kale <kalsac@amazon.com> * Use RemoteSegmentStoreDirectory instead of RemoteDirectory (#4240) * Use RemoteSegmentStoreDirectory instead of RemoteDirectory Signed-off-by: Sachin Kale <kalsac@amazon.com> Signed-off-by: Sachin Kale <kalsac@amazon.com> Co-authored-by: Sachin Kale <kalsac@amazon.com>
- Loading branch information
1 parent
ea5b946
commit 03273b5
Showing
58 changed files
with
3,627 additions
and
62 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
34 changes: 34 additions & 0 deletions
34
rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.restore.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| { | ||
| "remote_store.restore":{ | ||
| "documentation":{ | ||
| "url": "https://opensearch.org/docs/latest/opensearch/rest-api/remote-store#restore", | ||
| "description":"Restores from remote store." | ||
| }, | ||
| "stability":"experimental", | ||
| "url":{ | ||
| "paths":[ | ||
| { | ||
| "path":"/_remotestore/_restore", | ||
| "methods":[ | ||
| "POST" | ||
| ] | ||
| } | ||
| ] | ||
| }, | ||
| "params":{ | ||
| "cluster_manager_timeout":{ | ||
| "type":"time", | ||
| "description":"Explicit operation timeout for connection to cluster-manager node" | ||
| }, | ||
| "wait_for_completion":{ | ||
| "type":"boolean", | ||
| "description":"Should this request wait until the operation has completed before returning", | ||
| "default":false | ||
| } | ||
| }, | ||
| "body":{ | ||
| "description":"A comma separated list of index IDs", | ||
| "required":true | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
...ava/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.admin.cluster.remotestore.restore; | ||
|
|
||
| import org.opensearch.action.ActionType; | ||
|
|
||
| /** | ||
| * Restore remote store action | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public final class RestoreRemoteStoreAction extends ActionType<RestoreRemoteStoreResponse> { | ||
|
|
||
| public static final RestoreRemoteStoreAction INSTANCE = new RestoreRemoteStoreAction(); | ||
| public static final String NAME = "cluster:admin/remotestore/restore"; | ||
|
|
||
| private RestoreRemoteStoreAction() { | ||
| super(NAME, RestoreRemoteStoreResponse::new); | ||
| } | ||
| } |
183 changes: 183 additions & 0 deletions
183
...va/org/opensearch/action/admin/cluster/remotestore/restore/RestoreRemoteStoreRequest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.admin.cluster.remotestore.restore; | ||
|
|
||
| import org.opensearch.action.ActionRequestValidationException; | ||
| import org.opensearch.action.support.IndicesOptions; | ||
| import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; | ||
| import org.opensearch.common.Strings; | ||
| import org.opensearch.common.io.stream.StreamInput; | ||
| import org.opensearch.common.io.stream.StreamOutput; | ||
| import org.opensearch.common.xcontent.ToXContentObject; | ||
| import org.opensearch.common.xcontent.XContentBuilder; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.Arrays; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Objects; | ||
|
|
||
| import static org.opensearch.action.ValidateActions.addValidationError; | ||
|
|
||
| /** | ||
| * Restore remote store request | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public class RestoreRemoteStoreRequest extends ClusterManagerNodeRequest<RestoreRemoteStoreRequest> implements ToXContentObject { | ||
|
|
||
| private String[] indices = Strings.EMPTY_ARRAY; | ||
| private Boolean waitForCompletion; | ||
|
|
||
| public RestoreRemoteStoreRequest() {} | ||
|
|
||
| public RestoreRemoteStoreRequest(StreamInput in) throws IOException { | ||
| super(in); | ||
| indices = in.readStringArray(); | ||
| waitForCompletion = in.readOptionalBoolean(); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| super.writeTo(out); | ||
| out.writeStringArray(indices); | ||
| out.writeOptionalBoolean(waitForCompletion); | ||
| } | ||
|
|
||
| @Override | ||
| public ActionRequestValidationException validate() { | ||
| ActionRequestValidationException validationException = null; | ||
| if (indices == null || indices.length == 0) { | ||
| validationException = addValidationError("indices are missing", validationException); | ||
| } | ||
| return validationException; | ||
| } | ||
|
|
||
| /** | ||
| * Sets the list of indices that should be restored from the remote store | ||
| * <p> | ||
| * The list of indices supports multi-index syntax. For example: "+test*" ,"-test42" will index all indices with | ||
| * prefix "test" except index "test42". Aliases are not supported. An empty list or {"_all"} will restore all open | ||
| * indices in the cluster. | ||
| * | ||
| * @param indices list of indices | ||
| * @return this request | ||
| */ | ||
| public RestoreRemoteStoreRequest indices(String... indices) { | ||
| this.indices = indices; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Sets the list of indices that should be restored from the remote store | ||
| * <p> | ||
| * The list of indices supports multi-index syntax. For example: "+test*" ,"-test42" will index all indices with | ||
| * prefix "test" except index "test42". Aliases are not supported. An empty list or {"_all"} will restore all open | ||
| * indices in the cluster. | ||
| * | ||
| * @param indices list of indices | ||
| * @return this request | ||
| */ | ||
| public RestoreRemoteStoreRequest indices(List<String> indices) { | ||
| this.indices = indices.toArray(new String[indices.size()]); | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Returns list of indices that should be restored from the remote store | ||
| */ | ||
| public String[] indices() { | ||
| return indices; | ||
| } | ||
|
|
||
| /** | ||
| * If this parameter is set to true the operation will wait for completion of restore process before returning. | ||
| * | ||
| * @param waitForCompletion if true the operation will wait for completion | ||
| * @return this request | ||
| */ | ||
| public RestoreRemoteStoreRequest waitForCompletion(boolean waitForCompletion) { | ||
| this.waitForCompletion = waitForCompletion; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * Returns wait for completion setting | ||
| * | ||
| * @return true if the operation will wait for completion | ||
| */ | ||
| public boolean waitForCompletion() { | ||
| return waitForCompletion; | ||
| } | ||
|
|
||
| /** | ||
| * Parses restore definition | ||
| * | ||
| * @param source restore definition | ||
| * @return this request | ||
| */ | ||
| @SuppressWarnings("unchecked") | ||
| public RestoreRemoteStoreRequest source(Map<String, Object> source) { | ||
| for (Map.Entry<String, Object> entry : source.entrySet()) { | ||
| String name = entry.getKey(); | ||
| if (name.equals("indices")) { | ||
| if (entry.getValue() instanceof String) { | ||
| indices(Strings.splitStringByCommaToArray((String) entry.getValue())); | ||
| } else if (entry.getValue() instanceof ArrayList) { | ||
| indices((ArrayList<String>) entry.getValue()); | ||
| } else { | ||
| throw new IllegalArgumentException("malformed indices section, should be an array of strings"); | ||
| } | ||
| } else { | ||
| if (IndicesOptions.isIndicesOptions(name) == false) { | ||
| throw new IllegalArgumentException("Unknown parameter " + name); | ||
| } | ||
| } | ||
| } | ||
| return this; | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(); | ||
| builder.startArray("indices"); | ||
| for (String index : indices) { | ||
| builder.value(index); | ||
| } | ||
| builder.endArray(); | ||
| builder.endObject(); | ||
| return builder; | ||
| } | ||
|
|
||
| @Override | ||
| public String getDescription() { | ||
| return "remote_store"; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object o) { | ||
| if (this == o) return true; | ||
| if (o == null || getClass() != o.getClass()) return false; | ||
| RestoreRemoteStoreRequest that = (RestoreRemoteStoreRequest) o; | ||
| return waitForCompletion == that.waitForCompletion && Arrays.equals(indices, that.indices); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| int result = Objects.hash(waitForCompletion); | ||
| result = 31 * result + Arrays.hashCode(indices); | ||
| return result; | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return Strings.toString(this); | ||
| } | ||
| } |
Oops, something went wrong.