-
Notifications
You must be signed in to change notification settings - Fork 8.8k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
HDFS-17565. EC: dfs.datanode.ec.reconstruction.threads should support… #6928
base: trunk
Are you sure you want to change the base?
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -89,6 +89,8 @@ | |
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DISK_BALANCER_ENABLED_DEFAULT; | ||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DISK_BALANCER_PLAN_VALID_INTERVAL; | ||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DISK_BALANCER_PLAN_VALID_INTERVAL_DEFAULT; | ||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_THREADS_DEFAULT; | ||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_THREADS_KEY; | ||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_DEFAULT; | ||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_MAX_NUM_BLOCKS_TO_LOG_KEY; | ||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT; | ||
|
@@ -374,7 +376,8 @@ public class DataNode extends ReconfigurableBase | |
DFS_DATANODE_DATA_TRANSFER_BANDWIDTHPERSEC_KEY, | ||
DFS_DATANODE_DATA_WRITE_BANDWIDTHPERSEC_KEY, | ||
DFS_DATANODE_DATA_READ_BANDWIDTHPERSEC_KEY, | ||
DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY)); | ||
DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY, | ||
DFS_DN_EC_RECONSTRUCTION_THREADS_KEY)); | ||
|
||
public static final String METRICS_LOG_NAME = "DataNodeMetricsLog"; | ||
|
||
|
@@ -740,6 +743,8 @@ public String reconfigurePropertyImpl(String property, String newVal) | |
return reconfDiskBalancerParameters(property, newVal); | ||
case DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY: | ||
return reconfSlowIoWarningThresholdParameters(property, newVal); | ||
case DFS_DN_EC_RECONSTRUCTION_THREADS_KEY: | ||
return reconfStripedReconstructionParameters(property, newVal); | ||
default: | ||
break; | ||
} | ||
|
@@ -1079,6 +1084,20 @@ private String reconfSlowIoWarningThresholdParameters(String property, String ne | |
} | ||
} | ||
|
||
private String reconfStripedReconstructionParameters(String property, String newVal) | ||
throws ReconfigurationException { | ||
String result = null; | ||
try { | ||
int size = (newVal == null ? DFS_DN_EC_RECONSTRUCTION_THREADS_DEFAULT : | ||
Integer.parseInt(newVal)); | ||
result = Long.toString(size); | ||
ecWorker.setStripedReconstructionPoolSize(size); | ||
return result; | ||
} catch (IllegalArgumentException e) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Catch NumberFormatException alse. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @zeekling NumberFormatException inherits from IllegalArgumentException. So I think there are no need to catch NumberFormatException. |
||
throw new ReconfigurationException(property, newVal, getConf().get(property), e); | ||
} | ||
} | ||
|
||
/** | ||
* Get a list of the keys of the re-configurable properties in configuration. | ||
*/ | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,6 +17,7 @@ | |
*/ | ||
package org.apache.hadoop.hdfs.server.datanode.erasurecode; | ||
|
||
import org.apache.hadoop.classification.VisibleForTesting; | ||
import org.apache.hadoop.util.Preconditions; | ||
import org.apache.hadoop.classification.InterfaceAudience; | ||
import org.apache.hadoop.conf.Configuration; | ||
|
@@ -37,6 +38,9 @@ | |
import java.util.concurrent.TimeUnit; | ||
import java.util.concurrent.atomic.AtomicInteger; | ||
|
||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remove unused import. |
||
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_THREADS_KEY; | ||
|
||
/** | ||
* ErasureCodingWorker handles the erasure coding reconstruction work commands. | ||
* These commands would be issued from Namenode as part of Datanode's heart beat | ||
|
@@ -172,4 +176,19 @@ public void shutDown() { | |
public float getXmitWeight() { | ||
return xmitWeight; | ||
} | ||
|
||
public void setStripedReconstructionPoolSize(int size) { | ||
Preconditions.checkArgument(size > 0, | ||
DFS_DN_EC_RECONSTRUCTION_THREADS_KEY + " should be greater than 0"); | ||
this.stripedReconstructionPool.setCorePoolSize(size); | ||
this.stripedReconstructionPool.setMaximumPoolSize(size); | ||
} | ||
|
||
@VisibleForTesting | ||
public int getStripedReconstructionPoolSize() { | ||
int poolSize = this.stripedReconstructionPool.getCorePoolSize(); | ||
Preconditions.checkArgument(poolSize == this.stripedReconstructionPool.getMaximumPoolSize(), | ||
"The maximum pool size should be equal to core pool size"); | ||
return poolSize; | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
result = Integer.toString(size);
?