You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by di...@apache.org on 2021/02/14 05:12:51 UTC
[hadoop] branch trunk updated: HDFS-15814. Make some parameters
configurable for DataNodeDiskMetrics (#2676)
This is an automated email from the ASF dual-hosted git repository.
dineshc pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c3134ab HDFS-15814. Make some parameters configurable for DataNodeDiskMetrics (#2676)
c3134ab is described below
commit c3134ab3a99d4109d9ae3fcf216018351eb1d36f
Author: litao <55...@users.noreply.github.com>
AuthorDate: Sun Feb 14 13:12:24 2021 +0800
HDFS-15814. Make some parameters configurable for DataNodeDiskMetrics (#2676)
---
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +++++++
.../hadoop/hdfs/server/datanode/DataNode.java | 2 +-
.../datanode/metrics/DataNodeDiskMetrics.java | 26 +++++++++++++++++-----
.../src/main/resources/hdfs-default.xml | 16 +++++++++++++
4 files changed, 46 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index bf0ed14..d04a842 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -676,6 +676,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.datanode.slowpeer.low.threshold.ms";
public static final long DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT =
5L;
+ public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY =
+ "dfs.datanode.min.outlier.detection.disks";
+ public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT =
+ 5L;
+ public static final String DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY =
+ "dfs.datanode.slowdisk.low.threshold.ms";
+ public static final long DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT =
+ 20L;
public static final String DFS_DATANODE_HOST_NAME_KEY =
HdfsClientConfigKeys.DeprecatedKeys.DFS_DATANODE_HOST_NAME_KEY;
public static final String DFS_NAMENODE_CHECKPOINT_DIR_KEY =
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index b16fe44..742b815 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1489,7 +1489,7 @@ public class DataNode extends ReconfigurableBase
if (dnConf.diskStatsEnabled) {
diskMetrics = new DataNodeDiskMetrics(this,
- dnConf.outliersReportIntervalMs);
+ dnConf.outliersReportIntervalMs, getConf());
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
index e431bde..d0969f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.datanode.metrics;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
@@ -48,8 +50,6 @@ public class DataNodeDiskMetrics {
DataNodeDiskMetrics.class);
private DataNode dn;
- private final long MIN_OUTLIER_DETECTION_DISKS = 5;
- private final long SLOW_DISK_LOW_THRESHOLD_MS = 20;
private final long detectionInterval;
private volatile boolean shouldRun;
private OutlierDetector slowDiskDetector;
@@ -61,11 +61,27 @@ public class DataNodeDiskMetrics {
// code, status should not be overridden by daemon thread.
private boolean overrideStatus = true;
- public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) {
+ /**
+ * Minimum number of disks to run outlier detection.
+ */
+ private final long minOutlierDetectionDisks;
+ /**
+ * Threshold in milliseconds below which a disk is definitely not slow.
+ */
+ private final long lowThresholdMs;
+
+ public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs,
+ Configuration conf) {
this.dn = dn;
this.detectionInterval = diskOutlierDetectionIntervalMs;
- slowDiskDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_DISKS,
- SLOW_DISK_LOW_THRESHOLD_MS);
+ minOutlierDetectionDisks =
+ conf.getLong(DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
+ DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT);
+ lowThresholdMs =
+ conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY,
+ DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT);
+ slowDiskDetector =
+ new OutlierDetector(minOutlierDetectionDisks, lowThresholdMs);
shouldRun = true;
startDiskOutlierDetectionThread();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index e13e000..0c5bb35 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2371,6 +2371,22 @@
</property>
<property>
+ <name>dfs.datanode.min.outlier.detection.disks</name>
+ <value>5</value>
+ <description>
+ Minimum number of disks to run outlier detection.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.slowdisk.low.threshold.ms</name>
+ <value>20</value>
+ <description>
+ Threshold in milliseconds below which a disk is definitely not slow.
+ </description>
+</property>
+
+<property>
<name>hadoop.user.group.metrics.percentiles.intervals</name>
<value></value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org