You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/14 01:21:37 UTC
[49/50] hadoop git commit: HDFS-336. dfsadmin -report should report
number of blocks from datanode. Contributed by Bharat Viswanadham.
HDFS-336. dfsadmin -report should report number of blocks from datanode. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9714fc1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9714fc1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9714fc1d
Branch: refs/heads/HDFS-7240
Commit: 9714fc1dd48edb1c40d96d69ae82ed3b0fab7748
Parents: 39537b7
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Mar 13 16:39:17 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Mar 13 16:39:17 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 31 ++++++++++++++--
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 5 ++-
.../src/main/proto/hdfs.proto | 1 +
.../hdfs/server/namenode/FSNamesystem.java | 1 +
.../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 38 ++++++++++++++++++++
.../src/test/resources/testHDFSConf.xml | 4 +++
6 files changed, 77 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9714fc1d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index 0a8c915..c140d06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -56,6 +56,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
private List<String> dependentHostNames = new LinkedList<>();
private String upgradeDomain;
public static final DatanodeInfo[] EMPTY_ARRAY = {};
+ private int numBlocks;
// Datanode administrative states
public enum AdminStates {
@@ -106,6 +107,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.upgradeDomain = from.getUpgradeDomain();
this.lastBlockReportTime = from.getLastBlockReportTime();
this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
+ this.numBlocks = from.getNumBlocks();
}
protected DatanodeInfo(DatanodeID nodeID) {
@@ -123,6 +125,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.adminState = null;
this.lastBlockReportTime = 0L;
this.lastBlockReportMonotonic = 0L;
+ this.numBlocks = 0;
}
protected DatanodeInfo(DatanodeID nodeID, String location) {
@@ -139,7 +142,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
final long lastUpdate, final long lastUpdateMonotonic,
final int xceiverCount, final String networkLocation,
final AdminStates adminState, final String upgradeDomain,
- final long lastBlockReportTime, final long lastBlockReportMonotonic) {
+ final long lastBlockReportTime, final long lastBlockReportMonotonic,
+ final int blockCount) {
super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
ipcPort);
this.capacity = capacity;
@@ -157,6 +161,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.upgradeDomain = upgradeDomain;
this.lastBlockReportTime = lastBlockReportTime;
this.lastBlockReportMonotonic = lastBlockReportMonotonic;
+ this.numBlocks = blockCount;
}
/** Network location name. */
@@ -247,6 +252,13 @@ public class DatanodeInfo extends DatanodeID implements Node {
public long getLastUpdateMonotonic() { return lastUpdateMonotonic;}
/**
+ * @return Num of Blocks
+ */
+ public int getNumBlocks() {
+ return numBlocks;
+ }
+
+ /**
* Set lastUpdate monotonic time
*/
public void setLastUpdateMonotonic(long lastUpdateMonotonic) {
@@ -301,6 +313,11 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.xceiverCount = xceiverCount;
}
+ /** Sets number of blocks. */
+ public void setNumBlocks(int blockCount) {
+ this.numBlocks = blockCount;
+ }
+
/** network location */
@Override
public String getNetworkLocation() {return location;}
@@ -351,6 +368,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
float cacheUsedPercent = getCacheUsedPercent();
float cacheRemainingPercent = getCacheRemainingPercent();
String lookupName = NetUtils.getHostNameOfIP(getName());
+ int blockCount = getNumBlocks();
buffer.append("Name: ").append(getName());
if (lookupName != null) {
@@ -406,6 +424,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
.append(
lastBlockReportTime != 0 ? new Date(lastBlockReportTime) : "Never")
.append("\n");
+ buffer.append("Num of Blocks: ").append(blockCount).append("\n");
return buffer.toString();
}
@@ -680,6 +699,8 @@ public class DatanodeInfo extends DatanodeID implements Node {
private long nonDfsUsed = 0L;
private long lastBlockReportTime = 0L;
private long lastBlockReportMonotonic = 0L;
+ private int numBlocks;
+
public DatanodeInfoBuilder setFrom(DatanodeInfo from) {
this.capacity = from.getCapacity();
@@ -697,6 +718,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.upgradeDomain = from.getUpgradeDomain();
this.lastBlockReportTime = from.getLastBlockReportTime();
this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic();
+ this.numBlocks = from.getNumBlocks();
setNodeID(from);
return this;
}
@@ -823,13 +845,18 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.lastBlockReportMonotonic = time;
return this;
}
+ public DatanodeInfoBuilder setNumBlocks(int blockCount) {
+ this.numBlocks = blockCount;
+ return this;
+ }
public DatanodeInfo build() {
return new DatanodeInfo(ipAddr, hostName, datanodeUuid, xferPort,
infoPort, infoSecurePort, ipcPort, capacity, dfsUsed, nonDfsUsed,
remaining, blockPoolUsed, cacheCapacity, cacheUsed, lastUpdate,
lastUpdateMonotonic, xceiverCount, location, adminState,
- upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic);
+ upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic,
+ numBlocks);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9714fc1d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 3180f70..d9e7aa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -333,6 +333,7 @@ public class PBHelperClient {
.setAdminState(convert(info.getAdminState()))
.setLastBlockReportTime(info.getLastBlockReportTime())
.setLastBlockReportMonotonic(info.getLastBlockReportMonotonic())
+ .setNumBlocks(info.getNumBlocks())
.build();
return builder.build();
}
@@ -704,7 +705,9 @@ public class PBHelperClient {
.setLastBlockReportTime(di.hasLastBlockReportTime() ?
di.getLastBlockReportTime() : 0)
.setLastBlockReportMonotonic(di.hasLastBlockReportMonotonic() ?
- di.getLastBlockReportMonotonic() : 0);
+ di.getLastBlockReportMonotonic() : 0)
+ .setNumBlocks(di.getNumBlocks());
+
if (di.hasNonDfsUsed()) {
dinfo.setNonDfsUsed(di.getNonDfsUsed());
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9714fc1d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 405495f..29d0b4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -132,6 +132,7 @@ message DatanodeInfoProto {
optional string upgradeDomain = 14;
optional uint64 lastBlockReportTime = 15 [default = 0];
optional uint64 lastBlockReportMonotonic = 16 [default = 0];
+ optional uint32 numBlocks = 17 [default = 0];
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9714fc1d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e0ece35..868f669 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4338,6 +4338,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
for (int i=0; i<arr.length; i++) {
arr[i] = new DatanodeInfoBuilder().setFrom(results.get(i))
.build();
+ arr[i].setNumBlocks(results.get(i).numBlocks());
}
} finally {
readUnlock(operationName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9714fc1d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
index 7237c88..1365b1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
@@ -882,4 +882,42 @@ public class TestDFSAdmin {
assertEquals(-1, ToolRunner.run(dfsAdmin,
new String[]{"-setBalancerBandwidth", "-10m"}));
}
+
+ @Test(timeout = 300000L)
+ public void testCheckNumOfBlocksInReportCommand() throws Exception {
+ Configuration config = new Configuration();
+ config.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);
+ config.set(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, "3s");
+
+ int numOfDatanodes = 1;
+ MiniDFSCluster miniDFSCluster = new MiniDFSCluster.Builder(config)
+ .numDataNodes(numOfDatanodes).build();
+ try {
+ miniDFSCluster.waitActive();
+ DistributedFileSystem dfs = miniDFSCluster.getFileSystem();
+ Path path= new Path("/tmp.txt");
+
+ DatanodeInfo[] dn = dfs.getDataNodeStats();
+ assertEquals(dn.length, numOfDatanodes);
+ //Block count should be 0, as no files are created
+ assertEquals(dn[0].getNumBlocks(), 0);
+
+
+ //Create a file with 2 blocks
+ DFSTestUtil.createFile(dfs, path, 1024, (short) 1, 0);
+ int expectedBlockCount = 2;
+
+ //Wait for One Heartbeat
+ Thread.sleep(3 * 1000);
+
+ dn = dfs.getDataNodeStats();
+ assertEquals(dn.length, numOfDatanodes);
+
+ //Block count should be 2, as file is created with block count 2
+ assertEquals(dn[0].getNumBlocks(), expectedBlockCount);
+
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9714fc1d/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 5094183..a13c441 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -16551,6 +16551,10 @@
<expected-output>Last contact: [a-zA-Z]+ [a-zA-Z]+ [0-9]+ [0-9:]+ [A-Z\-\+\:0-9]+ [0-9]+</expected-output>
</comparator>
<comparator>
+ <type>RegexpComparator</type>
+ <expected-output>Num of Blocks: [0-9]+</expected-output>
+ </comparator>
+ <comparator>
<type>TokenComparator</type>
<expected-output>Live datanodes</expected-output>
</comparator>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org