You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/07/29 11:19:58 UTC
[1/2] hadoop git commit: HDFS-8670. Better to exclude decommissioned
nodes for namenode NodeUsage JMX (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 dff49aee0 -> a0238d85f
refs/heads/trunk 2a1d65619 -> 6374ee0db
HDFS-8670. Better to exclude decommissioned nodes for namenode NodeUsage JMX (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6374ee0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6374ee0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6374ee0d
Branch: refs/heads/trunk
Commit: 6374ee0db445e0a1c3462c19ddee345df740cfb3
Parents: 2a1d656
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jul 29 14:47:19 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jul 29 14:47:19 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSNamesystem.java | 6 ++
.../apache/hadoop/hdfs/TestDecommission.java | 98 ++++++++++++++++++++
3 files changed, 107 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6374ee0d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9a0c6da..cf03d3c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1093,6 +1093,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8785. TestDistributedFileSystem is failing in trunk. (Xiaoyu Yao)
+ HDFS-8670. Better to exclude decommissioned nodes for namenode NodeUsage JMX
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6374ee0d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 0b44431..a259070 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5999,6 +5999,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
new HashMap<String, Map<String,Object>>();
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
blockManager.getDatanodeManager().fetchDatanodes(live, null, true);
+ for (Iterator<DatanodeDescriptor> it = live.iterator(); it.hasNext();) {
+ DatanodeDescriptor node = it.next();
+ if (node.isDecommissionInProgress() || node.isDecommissioned()) {
+ it.remove();
+ }
+ }
if (live.size() > 0) {
float totalDfsUsed = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6374ee0d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index 8f965ad..413a3cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -28,6 +28,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.concurrent.ExecutionException;
@@ -56,6 +57,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DecommissionManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -68,6 +70,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
+import org.mortbay.util.ajax.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1127,4 +1130,99 @@ public class TestDecommission {
assertEquals("Unexpected number of pending nodes", pending,
decomManager.getNumPendingNodes());
}
+
+ /**
+ * Decommissioned node should not be considered while calculating node usage
+ * @throws InterruptedException
+ */
+ @Test
+ public void testNodeUsageAfterDecommissioned()
+ throws IOException, InterruptedException {
+ nodeUsageVerification(2, new long[] { 26384L, 26384L },
+ AdminStates.DECOMMISSIONED);
+ }
+
+ /**
+ * DECOMMISSION_INPROGRESS node should not be considered
+ * while calculating node usage
+ * @throws InterruptedException
+ */
+ @Test
+ public void testNodeUsageWhileDecommissioining()
+ throws IOException, InterruptedException {
+ nodeUsageVerification(1, new long[] { 26384L },
+ AdminStates.DECOMMISSION_INPROGRESS);
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ public void nodeUsageVerification(int numDatanodes, long[] nodesCapacity,
+ AdminStates decommissionState) throws IOException, InterruptedException {
+ Map<String, Map<String, String>> usage = null;
+ DatanodeInfo decommissionedNodeInfo = null;
+ String zeroNodeUsage = "0.00%";
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
+ FileSystem fileSys = null;
+ Path file1 = new Path("testNodeUsage.dat");
+ try {
+ SimulatedFSDataset.setFactory(conf);
+ cluster =
+ new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(1))
+ .numDataNodes(numDatanodes)
+ .simulatedCapacities(nodesCapacity).build();
+ cluster.waitActive();
+ DFSClient client = getDfsClient(cluster.getNameNode(0), conf);
+ validateCluster(client, numDatanodes);
+
+ ArrayList<ArrayList<DatanodeInfo>> namenodeDecomList =
+ new ArrayList<ArrayList<DatanodeInfo>>(1);
+ namenodeDecomList.add(0, new ArrayList<DatanodeInfo>(numDatanodes));
+
+ if (decommissionState == AdminStates.DECOMMISSIONED) {
+ // Move datanode1 to Decommissioned state
+ ArrayList<DatanodeInfo> decommissionedNode = namenodeDecomList.get(0);
+ decommissionedNodeInfo = decommissionNode(0, null,
+ decommissionedNode, decommissionState);
+ }
+ // Write a file(replica 1).Hence will be written to only one live node.
+ fileSys = cluster.getFileSystem(0);
+ FSNamesystem ns = cluster.getNamesystem(0);
+ writeFile(fileSys, file1, 1);
+ Thread.sleep(2000);
+
+ // min NodeUsage should not be 0.00%
+ usage = (Map<String, Map<String, String>>) JSON.parse(ns.getNodeUsage());
+ String minUsageBeforeDecom = usage.get("nodeUsage").get("min");
+ assertTrue(!minUsageBeforeDecom.equalsIgnoreCase(zeroNodeUsage));
+
+ if (decommissionState == AdminStates.DECOMMISSION_INPROGRESS) {
+ // Start decommissioning datanode
+ ArrayList<DatanodeInfo> decommissioningNodes = namenodeDecomList.
+ get(0);
+ decommissionedNodeInfo = decommissionNode(0, null,
+ decommissioningNodes, decommissionState);
+ // NodeUsage should not include DECOMMISSION_INPROGRESS node
+ // (minUsage should be 0.00%)
+ usage = (Map<String, Map<String, String>>)
+ JSON.parse(ns.getNodeUsage());
+ assertTrue(usage.get("nodeUsage").get("min").
+ equalsIgnoreCase(zeroNodeUsage));
+ }
+ // Recommission node
+ recommissionNode(0, decommissionedNodeInfo);
+
+ usage = (Map<String, Map<String, String>>) JSON.parse(ns.getNodeUsage());
+ String nodeusageAfterRecommi =
+ decommissionState == AdminStates.DECOMMISSION_INPROGRESS
+ ? minUsageBeforeDecom
+ : zeroNodeUsage;
+ assertTrue(usage.get("nodeUsage").get("min").
+ equalsIgnoreCase(nodeusageAfterRecommi));
+ } finally {
+ cleanupFile(fileSys, file1);
+ cluster.shutdown();
+ }
+ }
}
[2/2] hadoop git commit: HDFS-8670. Better to exclude decommissioned
nodes for namenode NodeUsage JMX (Contributed by J.Andreina)
Posted by vi...@apache.org.
HDFS-8670. Better to exclude decommissioned nodes for namenode NodeUsage JMX (Contributed by J.Andreina)
(cherry picked from commit 6374ee0db445e0a1c3462c19ddee345df740cfb3)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0238d85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0238d85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0238d85
Branch: refs/heads/branch-2
Commit: a0238d85f61f802c04aabcff598778f21ff67067
Parents: dff49ae
Author: Vinayakumar B <vi...@apache.org>
Authored: Wed Jul 29 14:47:19 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Wed Jul 29 14:49:16 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSNamesystem.java | 6 ++
.../apache/hadoop/hdfs/TestDecommission.java | 98 ++++++++++++++++++++
3 files changed, 107 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238d85/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 537c06d..4b7af3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -751,6 +751,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class.
(Surendra Singh Lilhore via aajisaka)
+ HDFS-8670. Better to exclude decommissioned nodes for namenode NodeUsage JMX
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238d85/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index f2adeb8..4f129f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5994,6 +5994,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
new HashMap<String, Map<String,Object>>();
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
blockManager.getDatanodeManager().fetchDatanodes(live, null, true);
+ for (Iterator<DatanodeDescriptor> it = live.iterator(); it.hasNext();) {
+ DatanodeDescriptor node = it.next();
+ if (node.isDecommissionInProgress() || node.isDecommissioned()) {
+ it.remove();
+ }
+ }
if (live.size() > 0) {
float totalDfsUsed = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238d85/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index 1f18014..d91cd2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -28,6 +28,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.concurrent.ExecutionException;
@@ -56,6 +57,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DecommissionManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -68,6 +70,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
+import org.mortbay.util.ajax.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1153,4 +1156,99 @@ public class TestDecommission {
assertEquals("Unexpected number of pending nodes", pending,
decomManager.getNumPendingNodes());
}
+
+ /**
+ * Decommissioned node should not be considered while calculating node usage
+ * @throws InterruptedException
+ */
+ @Test
+ public void testNodeUsageAfterDecommissioned()
+ throws IOException, InterruptedException {
+ nodeUsageVerification(2, new long[] { 26384L, 26384L },
+ AdminStates.DECOMMISSIONED);
+ }
+
+ /**
+ * DECOMMISSION_INPROGRESS node should not be considered
+ * while calculating node usage
+ * @throws InterruptedException
+ */
+ @Test
+ public void testNodeUsageWhileDecommissioining()
+ throws IOException, InterruptedException {
+ nodeUsageVerification(1, new long[] { 26384L },
+ AdminStates.DECOMMISSION_INPROGRESS);
+ }
+
+ @SuppressWarnings({ "unchecked" })
+ public void nodeUsageVerification(int numDatanodes, long[] nodesCapacity,
+ AdminStates decommissionState) throws IOException, InterruptedException {
+ Map<String, Map<String, String>> usage = null;
+ DatanodeInfo decommissionedNodeInfo = null;
+ String zeroNodeUsage = "0.00%";
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
+ FileSystem fileSys = null;
+ Path file1 = new Path("testNodeUsage.dat");
+ try {
+ SimulatedFSDataset.setFactory(conf);
+ cluster =
+ new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(1))
+ .numDataNodes(numDatanodes)
+ .simulatedCapacities(nodesCapacity).build();
+ cluster.waitActive();
+ DFSClient client = getDfsClient(cluster.getNameNode(0), conf);
+ validateCluster(client, numDatanodes);
+
+ ArrayList<ArrayList<DatanodeInfo>> namenodeDecomList =
+ new ArrayList<ArrayList<DatanodeInfo>>(1);
+ namenodeDecomList.add(0, new ArrayList<DatanodeInfo>(numDatanodes));
+
+ if (decommissionState == AdminStates.DECOMMISSIONED) {
+ // Move datanode1 to Decommissioned state
+ ArrayList<DatanodeInfo> decommissionedNode = namenodeDecomList.get(0);
+ decommissionedNodeInfo = decommissionNode(0, null,
+ decommissionedNode, decommissionState);
+ }
+ // Write a file(replica 1).Hence will be written to only one live node.
+ fileSys = cluster.getFileSystem(0);
+ FSNamesystem ns = cluster.getNamesystem(0);
+ writeFile(fileSys, file1, 1);
+ Thread.sleep(2000);
+
+ // min NodeUsage should not be 0.00%
+ usage = (Map<String, Map<String, String>>) JSON.parse(ns.getNodeUsage());
+ String minUsageBeforeDecom = usage.get("nodeUsage").get("min");
+ assertTrue(!minUsageBeforeDecom.equalsIgnoreCase(zeroNodeUsage));
+
+ if (decommissionState == AdminStates.DECOMMISSION_INPROGRESS) {
+ // Start decommissioning datanode
+ ArrayList<DatanodeInfo> decommissioningNodes = namenodeDecomList.
+ get(0);
+ decommissionedNodeInfo = decommissionNode(0, null,
+ decommissioningNodes, decommissionState);
+ // NodeUsage should not include DECOMMISSION_INPROGRESS node
+ // (minUsage should be 0.00%)
+ usage = (Map<String, Map<String, String>>)
+ JSON.parse(ns.getNodeUsage());
+ assertTrue(usage.get("nodeUsage").get("min").
+ equalsIgnoreCase(zeroNodeUsage));
+ }
+ // Recommission node
+ recommissionNode(0, decommissionedNodeInfo);
+
+ usage = (Map<String, Map<String, String>>) JSON.parse(ns.getNodeUsage());
+ String nodeusageAfterRecommi =
+ decommissionState == AdminStates.DECOMMISSION_INPROGRESS
+ ? minUsageBeforeDecom
+ : zeroNodeUsage;
+ assertTrue(usage.get("nodeUsage").get("min").
+ equalsIgnoreCase(nodeusageAfterRecommi));
+ } finally {
+ cleanupFile(fileSys, file1);
+ cluster.shutdown();
+ }
+ }
}