You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/04/21 00:37:10 UTC
[05/13] hadoop git commit: HDFS-9412. getBlocks occupies FSLock and
takes too long to complete. Contributed by He Tianyi.
HDFS-9412. getBlocks occupies FSLock and takes too long to complete. Contributed by He Tianyi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22855e6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22855e6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22855e6f
Branch: refs/heads/YARN-3368
Commit: 22855e6f2a6d6a4656da0802891b35323591ef71
Parents: 931cd28
Author: Walter Su <wa...@apache.org>
Authored: Mon Apr 18 09:28:02 2016 +0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 20 15:36:45 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/blockmanagement/BlockManager.java | 17 +++++++++++++++++
.../java/org/apache/hadoop/hdfs/TestGetBlocks.java | 8 ++++++--
2 files changed, 23 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22855e6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 104d723..8b50ef8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -305,6 +305,14 @@ public class BlockManager implements BlockStatsMXBean {
* processed again after aquiring lock again.
*/
private int numBlocksPerIteration;
+
+ /**
+ * Minimum size that a block can be sent to Balancer through getBlocks.
+ * And after HDFS-8824, the small blocks are unused anyway, so there's no
+ * point to send them to balancer.
+ */
+ private long getBlocksMinBlockSize = -1;
+
/**
* Progress of the Reconstruction queues initialisation.
*/
@@ -414,6 +422,9 @@ public class BlockManager implements BlockStatsMXBean {
this.numBlocksPerIteration = conf.getInt(
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT,
DFSConfigKeys.DFS_BLOCK_MISREPLICATION_PROCESSING_LIMIT_DEFAULT);
+ this.getBlocksMinBlockSize = conf.getLongBytes(
+ DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY,
+ DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT);
this.blockReportLeaseManager = new BlockReportLeaseManager(conf);
bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf);
@@ -1179,6 +1190,9 @@ public class BlockManager implements BlockStatsMXBean {
while(totalSize<size && iter.hasNext()) {
curBlock = iter.next();
if(!curBlock.isComplete()) continue;
+ if (curBlock.getNumBytes() < getBlocksMinBlockSize) {
+ continue;
+ }
totalSize += addBlock(curBlock, results);
}
if(totalSize<size) {
@@ -1186,6 +1200,9 @@ public class BlockManager implements BlockStatsMXBean {
for(int i=0; i<startBlock&&totalSize<size; i++) {
curBlock = iter.next();
if(!curBlock.isComplete()) continue;
+ if (curBlock.getNumBytes() < getBlocksMinBlockSize) {
+ continue;
+ }
totalSize += addBlock(curBlock, results);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22855e6f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
index 741e641..6e4b0f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
@@ -179,11 +179,15 @@ public class TestGetBlocks {
final int DEFAULT_BLOCK_SIZE = 1024;
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
+ CONF.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY,
+ DEFAULT_BLOCK_SIZE);
+
MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
REPLICATION_FACTOR).build();
try {
cluster.waitActive();
- long fileLen = 2 * DEFAULT_BLOCK_SIZE;
+ // the third block will not be visible to getBlocks
+ long fileLen = 2 * DEFAULT_BLOCK_SIZE + 1;
DFSTestUtil.createFile(cluster.getFileSystem(), new Path("/tmp.txt"),
fileLen, REPLICATION_FACTOR, 0L);
@@ -196,7 +200,7 @@ public class TestGetBlocks {
DFSUtilClient.getNNAddress(CONF), CONF);
locatedBlocks = dfsclient.getNamenode()
.getBlockLocations("/tmp.txt", 0, fileLen).getLocatedBlocks();
- assertEquals(2, locatedBlocks.size());
+ assertEquals(3, locatedBlocks.size());
notWritten = false;
for (int i = 0; i < 2; i++) {
dataNodes = locatedBlocks.get(i).getLocations();