You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by le...@apache.org on 2015/09/20 03:10:14 UTC
hadoop git commit: HDFS-9004. Add upgrade domain to DatanodeInfo.
Contributed by Ming Ma (via Lei (Eddy) Xu).
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a38dceda6 -> 0095936a6
HDFS-9004. Add upgrade domain to DatanodeInfo. Contributed by Ming Ma (via Lei (Eddy) Xu).
Change-Id: I887c66578eebd61acc34b94f18da6e6851c609f4
(cherry picked from commit 3a9c7076e81c1cc47c0ecf30c60abd9a65d8a501)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0095936a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0095936a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0095936a
Branch: refs/heads/branch-2
Commit: 0095936a6af0927ad6cb85818fab78b2241046e6
Parents: a38dced
Author: Lei Xu <le...@apache.org>
Authored: Sat Sep 19 18:08:09 2015 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Sat Sep 19 18:09:56 2015 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 41 ++++++++++++++++++--
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 3 ++
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 3 +-
.../src/main/proto/hdfs.proto | 1 +
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 3 ++
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 ++
.../server/namenode/TestNameNodeMXBean.java | 20 ++++++++++
8 files changed, 72 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0095936a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index 76a6757..0f294c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -53,7 +53,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
private String location = NetworkTopology.DEFAULT_RACK;
private String softwareVersion;
private List<String> dependentHostNames = new LinkedList<String>();
-
+ private String upgradeDomain;
// Datanode administrative states
public enum AdminStates {
@@ -95,6 +95,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.xceiverCount = from.getXceiverCount();
this.location = from.getNetworkLocation();
this.adminState = from.getAdminState();
+ this.upgradeDomain = from.getUpgradeDomain();
}
public DatanodeInfo(DatanodeID nodeID) {
@@ -120,12 +121,13 @@ public class DatanodeInfo extends DatanodeID implements Node {
final long capacity, final long dfsUsed, final long remaining,
final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
final long lastUpdate, final long lastUpdateMonotonic,
- final int xceiverCount, final AdminStates adminState) {
+ final int xceiverCount, final AdminStates adminState,
+ final String upgradeDomain) {
this(nodeID.getIpAddr(), nodeID.getHostName(), nodeID.getDatanodeUuid(),
nodeID.getXferPort(), nodeID.getInfoPort(), nodeID.getInfoSecurePort(),
nodeID.getIpcPort(), capacity, dfsUsed, remaining, blockPoolUsed,
cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic,
- xceiverCount, location, adminState);
+ xceiverCount, location, adminState, upgradeDomain);
}
/** Constructor */
@@ -137,6 +139,22 @@ public class DatanodeInfo extends DatanodeID implements Node {
final long lastUpdate, final long lastUpdateMonotonic,
final int xceiverCount, final String networkLocation,
final AdminStates adminState) {
+ this(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort, capacity, dfsUsed, remaining, blockPoolUsed, cacheCapacity,
+ cacheUsed, lastUpdate, lastUpdateMonotonic, xceiverCount,
+ networkLocation, adminState, null);
+ }
+
+ /** Constructor */
+ public DatanodeInfo(final String ipAddr, final String hostName,
+ final String datanodeUuid, final int xferPort, final int infoPort,
+ final int infoSecurePort, final int ipcPort,
+ final long capacity, final long dfsUsed, final long remaining,
+ final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
+ final long lastUpdate, final long lastUpdateMonotonic,
+ final int xceiverCount, final String networkLocation,
+ final AdminStates adminState,
+ final String upgradeDomain) {
super(ipAddr, hostName, datanodeUuid, xferPort, infoPort,
infoSecurePort, ipcPort);
this.capacity = capacity;
@@ -150,6 +168,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.xceiverCount = xceiverCount;
this.location = networkLocation;
this.adminState = adminState;
+ this.upgradeDomain = upgradeDomain;
}
/** Network location name */
@@ -298,6 +317,16 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.location = NodeBase.normalize(location);
}
+ /** Sets the upgrade domain */
+ public void setUpgradeDomain(String upgradeDomain) {
+ this.upgradeDomain = upgradeDomain;
+ }
+
+ /** upgrade domain */
+ public String getUpgradeDomain() {
+ return upgradeDomain;
+ }
+
/** Add a hostname to a list of network dependencies */
public void addDependentHostName(String hostname) {
dependentHostNames.add(hostname);
@@ -339,6 +368,9 @@ public class DatanodeInfo extends DatanodeID implements Node {
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
buffer.append("Rack: "+location+"\n");
}
+ if (upgradeDomain != null) {
+ buffer.append("Upgrade domain: "+ upgradeDomain +"\n");
+ }
buffer.append("Decommission Status : ");
if (isDecommissioned()) {
buffer.append("Decommissioned\n");
@@ -378,6 +410,9 @@ public class DatanodeInfo extends DatanodeID implements Node {
if (!NetworkTopology.DEFAULT_RACK.equals(location)) {
buffer.append(" "+location);
}
+ if (upgradeDomain != null) {
+ buffer.append(" " + upgradeDomain);
+ }
if (isDecommissioned()) {
buffer.append(" DD");
} else if (isDecommissionInProgress()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0095936a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 1e561cc..98de2e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -140,6 +140,9 @@ public class PBHelperClient {
if (info.getNetworkLocation() != null) {
builder.setLocation(info.getNetworkLocation());
}
+ if (info.getUpgradeDomain() != null) {
+ builder.setUpgradeDomain(info.getUpgradeDomain());
+ }
builder
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0095936a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index 713836c..3f85814 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -241,7 +241,8 @@ class JsonUtilClient {
getLong(m, "lastUpdateMonotonic", 0l),
getInt(m, "xceiverCount", 0),
getString(m, "networkLocation", ""),
- DatanodeInfo.AdminStates.valueOf(getString(m, "adminState", "NORMAL")));
+ DatanodeInfo.AdminStates.valueOf(getString(m, "adminState", "NORMAL")),
+ getString(m, "upgradeDomain", ""));
}
/** Convert an Object[] to a DatanodeInfo[]. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0095936a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 86fb462..ee77dc0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -98,6 +98,7 @@ message DatanodeInfoProto {
optional uint64 cacheCapacity = 11 [default = 0];
optional uint64 cacheUsed = 12 [default = 0];
optional uint64 lastUpdateMonotonic = 13 [default = 0];
+ optional string upgradeDomain = 14;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0095936a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index dd6bb59..06f2cf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -595,7 +595,8 @@ public class PBHelper {
di.getCapacity(), di.getDfsUsed(), di.getRemaining(),
di.getBlockPoolUsed(), di.getCacheCapacity(), di.getCacheUsed(),
di.getLastUpdate(), di.getLastUpdateMonotonic(),
- di.getXceiverCount(), PBHelper.convert(di.getAdminState()));
+ di.getXceiverCount(), PBHelper.convert(di.getAdminState()),
+ di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
}
static public DatanodeInfo[] convert(DatanodeInfoProto di[]) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0095936a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 52415a8..578b88c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5917,6 +5917,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.put("estimatedCapacityLostTotal",
volumeFailureSummary.getEstimatedCapacityLostTotal());
}
+ if (node.getUpgradeDomain() != null) {
+ innerinfo.put("upgradeDomain", node.getUpgradeDomain());
+ }
info.put(node.getHostName() + ":" + node.getXferPort(), innerinfo.build());
}
return JSON.toString(info);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0095936a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 6d04b2f..8428774 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -169,6 +169,9 @@ public class JsonUtil {
m.put("xceiverCount", datanodeinfo.getXceiverCount());
m.put("networkLocation", datanodeinfo.getNetworkLocation());
m.put("adminState", datanodeinfo.getAdminState().name());
+ if (datanodeinfo.getUpgradeDomain() != null) {
+ m.put("upgradeDomain", datanodeinfo.getUpgradeDomain());
+ }
return m;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0095936a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index f8aa317..f10d57e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
import org.apache.hadoop.io.nativeio.NativeIO;
@@ -76,6 +78,15 @@ public class TestNameNodeMXBean {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
cluster.waitActive();
+ // Set upgrade domain on the first DN.
+ String upgradeDomain = "abcd";
+ DatanodeManager dm = cluster.getNameNode().getNamesystem().
+ getBlockManager().getDatanodeManager();
+ DatanodeDescriptor dd = dm.getDatanode(
+ cluster.getDataNodes().get(0).getDatanodeId());
+ dd.setUpgradeDomain(upgradeDomain);
+ String dnXferAddrWithUpgradeDomainSet = dd.getXferAddr();
+
FSNamesystem fsn = cluster.getNameNode().namesystem;
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
@@ -125,6 +136,15 @@ public class TestNameNodeMXBean {
assertTrue(((Long)liveNode.get("capacity")) > 0);
assertTrue(liveNode.containsKey("numBlocks"));
assertTrue(((Long)liveNode.get("numBlocks")) == 0);
+ // a. By default the upgrade domain isn't defined on any DN.
+ // b. If the upgrade domain is set on a DN, JMX should have the same
+ // value.
+ String xferAddr = (String)liveNode.get("xferaddr");
+ if (!xferAddr.equals(dnXferAddrWithUpgradeDomainSet)) {
+ assertTrue(!liveNode.containsKey("upgradeDomain"));
+ } else {
+ assertTrue(liveNode.get("upgradeDomain").equals(upgradeDomain));
+ }
}
assertEquals(fsn.getLiveNodes(), alivenodeinfo);
// get attribute deadnodeinfo