Skip to content

Commit

Permalink
Moving path strategy method to Utils
Browse files Browse the repository at this point in the history 
Signed-off-by: Shourya Dutta Biswas <114977491+shourya035@users.noreply.github.com>
  • Loading branch information
@shourya035
shourya035 committed Apr 30, 2024
1 parent 4cf8f6e commit e4ce087
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 39 deletions.
35 changes: 5 additions & 30 deletions server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,16 @@
package org.opensearch.index.remote;

import org.apache.logging.log4j.Logger;
import org.opensearch.Version;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.routing.IndexRoutingTable;
import org.opensearch.cluster.routing.RoutingTable;
import org.opensearch.cluster.routing.ShardRouting;
import org.opensearch.cluster.routing.ShardRoutingState;
import org.opensearch.common.settings.Settings;
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
import org.opensearch.indices.replication.common.ReplicationType;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand All @@ -32,9 +28,8 @@
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStorePathStrategyDuringMigration;
import static org.opensearch.index.remote.RemoteStoreUtils.getRemoteStoreRepoName;
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING;
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;

Expand Down Expand Up @@ -135,35 +130,15 @@ private boolean needsRemoteIndexSettingsUpdate(
public void maybeUpdateRemoteStorePathStrategy(IndexMetadata.Builder indexMetadataBuilder, String index) {
if (indexHasRemotePathMetadata(indexMetadata) == false) {
logger.info("Adding remote store path strategy for index [{}] during migration", index);
indexMetadataBuilder.putCustom(REMOTE_STORE_CUSTOM_KEY, createRemoteStorePathTypeMetadata(clusterSettings, discoveryNodes));
indexMetadataBuilder.putCustom(
REMOTE_STORE_CUSTOM_KEY,
determineRemoteStorePathStrategyDuringMigration(clusterSettings, discoveryNodes)
);
} else {
logger.debug("Index {} already has remote store path strategy", index);
}
}

/**
* Generates the remote store path type information to be added to custom data of index metadata.
*
* @param clusterSettings Current Cluster settings from {@link ClusterState}
* @param discoveryNodes Current {@link DiscoveryNodes} from the cluster state
* @return {@link Map} to be added as custom data in index metadata
*/
private Map<String, String> createRemoteStorePathTypeMetadata(Settings clusterSettings, DiscoveryNodes discoveryNodes) {
Version minNodeVersion = discoveryNodes.getMinNodeVersion();
PathType pathType = Version.CURRENT.compareTo(minNodeVersion) <= 0
? CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.get(clusterSettings)
: PathType.FIXED;
PathHashAlgorithm pathHashAlgorithm = pathType == PathType.FIXED
? null
: CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.get(clusterSettings);
Map<String, String> remoteCustomData = new HashMap<>();
remoteCustomData.put(PathType.NAME, pathType.name());
if (Objects.nonNull(pathHashAlgorithm)) {
remoteCustomData.put(PathHashAlgorithm.NAME, pathHashAlgorithm.name());
}
return remoteCustomData;
}

public static boolean indexHasAllRemoteStoreRelatedMetadata(IndexMetadata indexMetadata) {
return indexHasRemoteStoreSettings(indexMetadata.getSettings()) && indexHasRemotePathMetadata(indexMetadata);
}
Expand Down
53 changes: 44 additions & 9 deletions server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.Version;
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.collect.Tuple;
import org.opensearch.common.settings.Settings;
import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;

import java.nio.ByteBuffer;
Expand All @@ -23,10 +26,13 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;

import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING;
import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;

/**
* Utils for remote store
*
Expand Down Expand Up @@ -156,22 +162,51 @@ static String longToCompositeBase64AndBinaryEncoding(long value, int len) {
return URL_BASE64_CHARSET[base64DecimalValue] + binaryPart;
}

/**
* Determines the remote store path strategy by reading the custom data map in IndexMetadata class.
*/
public static RemoteStorePathStrategy determineRemoteStorePathStrategy(IndexMetadata indexMetadata) {
Map<String, String> remoteCustomData = indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
assert remoteCustomData == null || remoteCustomData.containsKey(RemoteStoreEnums.PathType.NAME);
if (remoteCustomData != null && remoteCustomData.containsKey(RemoteStoreEnums.PathType.NAME)) {
RemoteStoreEnums.PathType pathType = RemoteStoreEnums.PathType.parseString(
remoteCustomData.get(RemoteStoreEnums.PathType.NAME)
remoteCustomData.get(RemoteStoreEnums.PathType.NAME)
);
String hashAlgoStr = remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME);
RemoteStoreEnums.PathHashAlgorithm hashAlgorithm = Objects.nonNull(hashAlgoStr)
? RemoteStoreEnums.PathHashAlgorithm.parseString(hashAlgoStr)
: null;
? RemoteStoreEnums.PathHashAlgorithm.parseString(hashAlgoStr)
: null;
return new RemoteStorePathStrategy(pathType, hashAlgorithm);
}
return new RemoteStorePathStrategy(RemoteStoreEnums.PathType.FIXED);
}

/**
* Generates the remote store path type information to be added to custom data of index metadata during migration
*
* @param clusterSettings Current Cluster settings from {@link ClusterState}
* @param discoveryNodes Current {@link DiscoveryNodes} from the cluster state
* @return {@link Map} to be added as custom data in index metadata
*/
public static Map<String, String> determineRemoteStorePathStrategyDuringMigration(
Settings clusterSettings,
DiscoveryNodes discoveryNodes
) {
Version minNodeVersion = discoveryNodes.getMinNodeVersion();
RemoteStoreEnums.PathType pathType = Version.CURRENT.compareTo(minNodeVersion) <= 0
? CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.get(clusterSettings)
: RemoteStoreEnums.PathType.FIXED;
RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm = pathType == RemoteStoreEnums.PathType.FIXED
? null
: CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.get(clusterSettings);
Map<String, String> remoteCustomData = new HashMap<>();
remoteCustomData.put(RemoteStoreEnums.PathType.NAME, pathType.name());
if (Objects.nonNull(pathHashAlgorithm)) {
remoteCustomData.put(RemoteStoreEnums.PathHashAlgorithm.NAME, pathHashAlgorithm.name());
}
return remoteCustomData;
}

/**
* Fetches segment and translog repository names from remote store node attributes.
* Returns a blank {@link HashMap} if the cluster does not contain any remote nodes.
Expand All @@ -183,10 +218,10 @@ public static RemoteStorePathStrategy determineRemoteStorePathStrategy(IndexMeta
*/
public static Map<String, String> getRemoteStoreRepoName(DiscoveryNodes discoveryNodes) {
Optional<DiscoveryNode> remoteNode = discoveryNodes.getNodes()
.values()
.stream()
.filter(DiscoveryNode::isRemoteStoreNode)
.findFirst();
.values()
.stream()
.filter(DiscoveryNode::isRemoteStoreNode)
.findFirst();
return remoteNode.map(RemoteStoreNodeAttribute::getDataRepoNames).orElseGet(HashMap::new);
}
}

0 comments on commit e4ce087

Please sign in to comment.