You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2016/06/24 16:36:15 UTC
hadoop git commit: Revert "HDFS-10534. NameNode WebUI should display
DataNode usage rate with a certain percentile. Contributed by Kai Sasaki."
Repository: hadoop
Updated Branches:
refs/heads/trunk 0424056a7 -> 22be604fa
Revert "HDFS-10534. NameNode WebUI should display DataNode usage rate with a certain percentile. Contributed by Kai Sasaki."
This reverts commit 0424056a77002f4a2334ee2eb240fbc67b676471.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22be604f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22be604f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22be604f
Branch: refs/heads/trunk
Commit: 22be604fa3e4054a8f329296be7334c1531b874f
Parents: 0424056
Author: Zhe Zhang <zh...@apache.org>
Authored: Fri Jun 24 09:36:03 2016 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Fri Jun 24 09:36:03 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 -----
.../hdfs/server/namenode/FSNamesystem.java | 21 +-------------------
.../src/main/resources/hdfs-default.xml | 11 ----------
.../src/main/webapps/hdfs/dfshealth.html | 4 ++--
.../server/namenode/TestNameNodeMXBean.java | 10 ----------
5 files changed, 3 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 9371f56..6640ec6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -451,11 +451,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.metrics.logger.period.seconds";
public static final int DFS_NAMENODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT =
600;
- public static final String DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE =
- "dfs.namenode.metrics.node-usage.percentile";
- public static final double DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE_DEFAULT
- = 0.95;
-
public static final String DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_KEY =
"dfs.datanode.metrics.logger.period.seconds";
public static final int DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 02aaeda..915ae97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -66,8 +66,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CAC
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
@@ -530,8 +528,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
private INodeAttributeProvider inodeAttributeProvider;
- private final double percentileFactor;
-
/**
* If the NN is in safemode, and not due to manual / low resources, we
* assume it must be because of startup. If the NN had low resources during
@@ -828,15 +824,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
alwaysUseDelegationTokensForTests = conf.getBoolean(
DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT);
-
- this.percentileFactor = conf.getDouble(
- DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE,
- DFS_NAMENODE_METRICS_NODE_USAGE_PERCENTILE_DEFAULT);
-
- Preconditions.checkArgument(0.0 < this.percentileFactor
- && this.percentileFactor <= 1.0, "Node usage percentile " +
- "factor must be between 0 and 1.");
-
+
this.dtSecretManager = createDelegationTokenSecretManager(conf);
this.dir = new FSDirectory(this, conf);
this.snapshotManager = new SnapshotManager(dir);
@@ -5626,7 +5614,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
float max = 0;
float min = 0;
float dev = 0;
- float percentile = 0;
final Map<String, Map<String,Object>> info =
new HashMap<String, Map<String,Object>>();
@@ -5652,7 +5639,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
median = usages[usages.length / 2];
max = usages[usages.length - 1];
min = usages[0];
- percentile = usages[(int)((usages.length - 1) * percentileFactor)];
for (i = 0; i < usages.length; i++) {
dev += (usages[i] - totalDfsUsed) * (usages[i] - totalDfsUsed);
@@ -5665,11 +5651,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
innerInfo.put("median", StringUtils.format("%.2f%%", median));
innerInfo.put("max", StringUtils.format("%.2f%%", max));
innerInfo.put("stdDev", StringUtils.format("%.2f%%", dev));
- final Map<String, Object> percentileInfo = new HashMap<String, Object>();
- percentileInfo.put("name", StringUtils.format("%dth percentile",
- (int)(percentileFactor * 100)));
- percentileInfo.put("value", StringUtils.format("%.2f%%", percentile));
- innerInfo.put("percentile", percentileInfo);
info.put("nodeUsage", innerInfo);
return JSON.toString(info);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index bc56dc8..856e6b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1897,17 +1897,6 @@
</property>
<property>
- <name>dfs.namenode.metrics.node-usage.percentile</name>
- <value>0.95</value>
- <description>
- This setting specifies the percentile level to report node usage metrics.
- For example 0.95 means reporting the node usage for the 95th percentile
- of all DataNodes. If this setting is at 0.95 and the reported node usage
- is 70%, it means 95% of DataNodes have a usage below 70%.
- </description>
-</property>
-
-<property>
<name>dfs.datanode.metrics.logger.period.seconds</name>
<value>600</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index d9a1f07..4fa2e4c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -166,8 +166,8 @@
<tr><th> Non DFS Used:</th><td>{NonDfsUsedSpace|fmt_bytes}</td></tr>
<tr><th> DFS Remaining:</th><td>{Free|fmt_bytes} ({PercentRemaining|fmt_percentage})</td></tr>
<tr><th> Block Pool Used:</th><td>{BlockPoolUsedSpace|fmt_bytes} ({PercentBlockPoolUsed|fmt_percentage})</td></tr>
- <tr><th> {#NodeUsage.nodeUsage}DataNodes usages% (Min/Median/Max/stdDev/{percentile.name}):{/NodeUsage.nodeUsage} </th>
- <td>{#NodeUsage.nodeUsage}{min} / {median} / {max} / {stdDev} / {percentile.value}{/NodeUsage.nodeUsage}</td></tr>
+ <tr><th> DataNodes usages% (Min/Median/Max/stdDev): </th>
+ <td>{#NodeUsage.nodeUsage}{min} / {median} / {max} / {stdDev}{/NodeUsage.nodeUsage}</td></tr>
{/nn}
{#fs}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22be604f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 3807076..dc8bea7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -173,16 +173,6 @@ public class TestNameNodeMXBean {
String nodeUsage = (String) (mbs.getAttribute(mxbeanName,
"NodeUsage"));
assertEquals("Bad value for NodeUsage", fsn.getNodeUsage(), nodeUsage);
- Map<String, Map<String, Object>> usage
- = (Map<String, Map<String, Object>>)JSON.parse(nodeUsage);
- assertTrue(usage.get("nodeUsage").containsKey("min"));
- assertTrue(usage.get("nodeUsage").containsKey("median"));
- assertTrue(usage.get("nodeUsage").containsKey("max"));
- assertTrue(usage.get("nodeUsage").containsKey("percentile"));
- Map<String, Object> percentileInfo
- = (Map<String, Object>)usage.get("nodeUsage").get("percentile");
- assertTrue(percentileInfo.containsKey("name"));
- assertTrue(percentileInfo.containsKey("value"));
// get attribute NameJournalStatus
String nameJournalStatus = (String) (mbs.getAttribute(mxbeanName,
"NameJournalStatus"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org