You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ta...@apache.org on 2018/09/28 15:54:20 UTC
hadoop git commit: HDFS-8196. Post enabled Erasure Coding Policies on
NameNode UI. Contributed by Kitti Nanasi and Kai Sasaki.
Repository: hadoop
Updated Branches:
refs/heads/trunk 8598b498b -> e2113500d
HDFS-8196. Post enabled Erasure Coding Policies on NameNode UI. Contributed by Kitti Nanasi and Kai Sasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2113500
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2113500
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2113500
Branch: refs/heads/trunk
Commit: e2113500df2479dd2d8f1d8a37fdb2bb16463181
Parents: 8598b49
Author: Takanobu Asanuma <ta...@apache.org>
Authored: Sat Sep 29 00:53:35 2018 +0900
Committer: Takanobu Asanuma <ta...@apache.org>
Committed: Sat Sep 29 00:53:35 2018 +0900
----------------------------------------------------------------------
.../namenode/ErasureCodingPolicyManager.java | 6 +++
.../hdfs/server/namenode/FSNamesystem.java | 9 ++++
.../namenode/metrics/ECBlockGroupsMBean.java | 5 ++
.../src/main/webapps/hdfs/dfshealth.html | 4 +-
.../server/namenode/TestNameNodeMXBean.java | 49 ++++++++++++++++++++
5 files changed, 72 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index e7de05b..d2bf3af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
+import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -430,4 +431,9 @@ public final class ErasureCodingPolicyManager {
allPolicies =
policiesByName.values().toArray(new ErasureCodingPolicyInfo[0]);
}
+
+ public String getEnabledPoliciesMetric() {
+ return StringUtils.join(", ",
+ enabledPoliciesByName.keySet());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 1caf7c2..137dfb9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4973,6 +4973,15 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return blockManager.getTotalECBlockGroups();
}
+ /**
+ * Get the enabled erasure coding policies separated with comma.
+ */
+ @Override // ECBlockGroupsMBean
+ @Metric({"EnabledEcPolicies", "Enabled erasure coding policies"})
+ public String getEnabledEcPolicies() {
+ return getErasureCodingPolicyManager().getEnabledPoliciesMetric();
+ }
+
@Override
public long getBlockDeletionStartTime() {
return startTime + blockManager.getStartupDelayBlockDeletionInMs();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
index 808fc62..e5d2a09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/ECBlockGroupsMBean.java
@@ -61,4 +61,9 @@ public interface ECBlockGroupsMBean {
* Return total number of erasure coded block groups.
*/
long getTotalECBlockGroups();
+
+ /**
+ * @return the enabled erasure coding policies separated with comma.
+ */
+ String getEnabledEcPolicies();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 4495b99..f993ae7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -186,10 +186,12 @@
{/eq}
<tr><th>Block Deletion Start Time</th><td>{BlockDeletionStartTime|date_tostring}</td></tr>
{/fs}
-
{#fsn}
<tr><th>Last Checkpoint Time</th><td>{@if cond="{LastCheckpointTime} === 0"}Never{:else}{LastCheckpointTime|date_tostring}{/if}</td></tr>
{/fsn}
+{#ecstat}
+ <tr><th>Enabled Erasure Coding Policies</th><td>{EnabledEcPolicies}</td></tr>
+{/ecstat}
</table>
<div class="page-header"><h1>NameNode Journal Status</h1></div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2113500/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 3728420..07a4945 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
@@ -729,6 +730,46 @@ public class TestNameNodeMXBean {
}
@Test
+ public void testEnabledEcPoliciesMetric() throws Exception {
+ MiniDFSCluster cluster = null;
+ DistributedFileSystem fs = null;
+ try {
+ Configuration conf = new HdfsConfiguration();
+
+ ErasureCodingPolicy defaultPolicy =
+ StripedFileTestUtil.getDefaultECPolicy();
+ int dataBlocks = defaultPolicy.getNumDataUnits();
+ int parityBlocks = defaultPolicy.getNumParityUnits();
+ int totalSize = dataBlocks + parityBlocks;
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(totalSize).build();
+ fs = cluster.getFileSystem();
+
+ final String defaultPolicyName = defaultPolicy.getName();
+ final String rs104PolicyName = "RS-10-4-1024k";
+
+ assertEquals("Enabled EC policies metric should return with " +
+ "the default EC policy", defaultPolicyName,
+ getEnabledEcPoliciesMetric());
+
+ fs.enableErasureCodingPolicy(rs104PolicyName);
+ assertEquals("Enabled EC policies metric should return with " +
+ "both enabled policies separated by a comma",
+ rs104PolicyName + ", " + defaultPolicyName,
+ getEnabledEcPoliciesMetric());
+
+ fs.disableErasureCodingPolicy(defaultPolicyName);
+ fs.disableErasureCodingPolicy(rs104PolicyName);
+ assertEquals("Enabled EC policies metric should return with " +
+ "an empty string if there is no enabled policy",
+ "", getEnabledEcPoliciesMetric());
+ } finally {
+ fs.close();
+ cluster.shutdown();
+ }
+ }
+
+ @Test
public void testVerifyMissingBlockGroupsMetrics() throws Exception {
MiniDFSCluster cluster = null;
DistributedFileSystem fs = null;
@@ -968,4 +1009,12 @@ public class TestNameNodeMXBean {
assertEquals("Unexpected total ec block groups!",
expectedTotalECBlockGroups, totalECBlockGroups.longValue());
}
+
+ private String getEnabledEcPoliciesMetric() throws Exception {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanName = new ObjectName(
+ "Hadoop:service=NameNode,name=ECBlockGroupsState");
+ return (String) (mbs.getAttribute(mxbeanName,
+ "EnabledEcPolicies"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org