You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2016/10/07 18:03:13 UTC
hadoop git commit: HDFS-8824. Do not use small blocks for balancing
the cluster.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 47fcae7da -> b23b60e7b
HDFS-8824. Do not use small blocks for balancing the cluster.
(cherry-picked from commit 61b9e5f7ff15daa0efd09e98efd70351f474c8cb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b23b60e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b23b60e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b23b60e7
Branch: refs/heads/branch-2.7
Commit: b23b60e7b445f42a1ebed9b7f203566079007704
Parents: 47fcae7
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Fri Aug 14 13:03:19 2015 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Fri Oct 7 11:01:29 2016 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +++
.../hadoop/hdfs/server/balancer/Balancer.java | 9 ++++-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 38 ++++++++++++--------
.../hdfs/server/balancer/TestBalancer.java | 10 ++++--
5 files changed, 46 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3350509..19e2bff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -50,6 +50,8 @@ Release 2.7.4 - UNRELEASED
HDFS-10745. Directly resolve paths into INodesInPath.
(Daryn Sharp via kihwal)
+ HDFS-8824. Do not use small blocks for balancing the cluster. (szetszwo)
+
OPTIMIZATIONS
HDFS-10896. Move lock logging logic from FSNamesystem into FSNamesystemLock.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 59df45c..e569fe9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -450,6 +450,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_BALANCER_ADDRESS_DEFAULT= "0.0.0.0:0";
public static final String DFS_BALANCER_KEYTAB_FILE_KEY = "dfs.balancer.keytab.file";
public static final String DFS_BALANCER_KERBEROS_PRINCIPAL_KEY = "dfs.balancer.kerberos.principal";
+ public static final String DFS_BALANCER_GETBLOCKS_SIZE_KEY = "dfs.balancer.getBlocks.size";
+ public static final long DFS_BALANCER_GETBLOCKS_SIZE_DEFAULT = 2L*1024*1024*1024; // 2GB
+ public static final String DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY = "dfs.balancer.getBlocks.min-block-size";
+ public static final long DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT = 10L*1024*1024; // 10MB
public static final String DFS_MOVER_MOVEDWINWIDTH_KEY = "dfs.mover.movedWinWidth";
public static final long DFS_MOVER_MOVEDWINWIDTH_DEFAULT = 5400*1000L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 3a31ded..1772475 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -249,9 +249,16 @@ public class Balancer {
DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY,
DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT);
+ final long getBlocksSize = getLong(conf,
+ DFSConfigKeys.DFS_BALANCER_GETBLOCKS_SIZE_KEY,
+ DFSConfigKeys.DFS_BALANCER_GETBLOCKS_SIZE_DEFAULT);
+ final long getBlocksMinBlockSize = getLong(conf,
+ DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY,
+ DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_DEFAULT);
+
this.dispatcher = new Dispatcher(theblockpool, p.nodesToBeIncluded,
p.nodesToBeExcluded, movedWinWidth, moverThreads, dispatcherThreads,
- maxConcurrentMovesPerNode, conf);
+ maxConcurrentMovesPerNode, getBlocksSize, getBlocksMinBlockSize, conf);
this.threshold = p.threshold;
this.policy = p.policy;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index e175330..c336cd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -85,9 +85,6 @@ import com.google.common.base.Preconditions;
public class Dispatcher {
static final Log LOG = LogFactory.getLog(Dispatcher.class);
- private static final long GB = 1L << 30; // 1GB
- private static final long MAX_BLOCKS_SIZE_TO_FETCH = 2 * GB;
-
private static final int MAX_NO_PENDING_MOVE_ITERATIONS = 5;
/**
* the period of time to delay the usage of a DataNode after hitting
@@ -122,6 +119,9 @@ public class Dispatcher {
/** The maximum number of concurrent blocks moves at a datanode */
private final int maxConcurrentMovesPerNode;
+ private final long getBlocksSize;
+ private final long getBlocksMinBlockSize;
+
static class Allocator {
private final int max;
private int count = 0;
@@ -645,8 +645,9 @@ public class Dispatcher {
* @return the total size of the received blocks in the number of bytes.
*/
private long getBlockList() throws IOException {
- final long size = Math.min(MAX_BLOCKS_SIZE_TO_FETCH, blocksToReceive);
+ final long size = Math.min(getBlocksSize, blocksToReceive);
final BlocksWithLocations newBlocks = nnc.getBlocks(getDatanodeInfo(), size);
+
if (LOG.isTraceEnabled()) {
LOG.trace("getBlocks(" + getDatanodeInfo() + ", "
+ StringUtils.TraditionalBinaryPrefix.long2String(size, "B", 2)
@@ -655,6 +656,11 @@ public class Dispatcher {
long bytesReceived = 0;
for (BlockWithLocations blk : newBlocks.getBlocks()) {
+ // Skip small blocks.
+ if (blk.getBlock().getNumBytes() < getBlocksMinBlockSize) {
+ continue;
+ }
+
bytesReceived += blk.getBlock().getNumBytes();
synchronized (globalBlocks) {
final DBlock block = globalBlocks.get(blk.getBlock());
@@ -833,9 +839,19 @@ public class Dispatcher {
}
}
+ /** Constructor called by Mover. */
public Dispatcher(NameNodeConnector nnc, Set<String> includedNodes,
Set<String> excludedNodes, long movedWinWidth, int moverThreads,
int dispatcherThreads, int maxConcurrentMovesPerNode, Configuration conf) {
+ this(nnc, includedNodes, excludedNodes, movedWinWidth,
+ moverThreads, dispatcherThreads, maxConcurrentMovesPerNode,
+ 0L, 0L, conf);
+ }
+
+ Dispatcher(NameNodeConnector nnc, Set<String> includedNodes,
+ Set<String> excludedNodes, long movedWinWidth, int moverThreads,
+ int dispatcherThreads, int maxConcurrentMovesPerNode,
+ long getBlocksSize, long getBlocksMinBlockSize, Configuration conf) {
this.nnc = nnc;
this.excludedNodes = excludedNodes;
this.includedNodes = includedNodes;
@@ -848,6 +864,9 @@ public class Dispatcher {
this.moverThreadAllocator = new Allocator(moverThreads);
this.maxConcurrentMovesPerNode = maxConcurrentMovesPerNode;
+ this.getBlocksSize = getBlocksSize;
+ this.getBlocksMinBlockSize = getBlocksMinBlockSize;
+
this.saslClient = new SaslDataTransferClient(conf,
DataTransferSaslUtil.getSaslPropertiesResolver(conf),
TrustedChannelResolver.getInstance(conf), nnc.fallbackToSimpleAuth);
@@ -995,9 +1014,6 @@ public class Dispatcher {
return getBytesMoved() - bytesLastMoved;
}
- /** The sleeping period before checking if block move is completed again */
- static private long blockMoveWaitTime = 30000L;
-
/**
* Wait for all block move confirmations.
* @return true if there is failed move execution
@@ -1019,7 +1035,7 @@ public class Dispatcher {
return hasFailure; // all pending queues are empty
}
try {
- Thread.sleep(blockMoveWaitTime);
+ Thread.sleep(1000);
} catch (InterruptedException ignored) {
}
}
@@ -1134,12 +1150,6 @@ public class Dispatcher {
movedBlocks.cleanup();
}
- /** set the sleeping period for block move completion check */
- @VisibleForTesting
- public static void setBlockMoveWaitTime(long time) {
- blockMoveWaitTime = time;
- }
-
@VisibleForTesting
public static void setDelayAfterErrors(long time) {
delayAfterErrors = time;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b23b60e7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 065e8a1..2ba5d55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -205,8 +205,6 @@ public class TestBalancer {
}
public static void initTestSetup() {
- Dispatcher.setBlockMoveWaitTime(1000L) ;
-
// do not create id file since it occupies the disk space
NameNodeConnector.setWrite2IdFile(false);
}
@@ -215,9 +213,12 @@ public class TestBalancer {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_BLOCK_SIZE);
conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DEFAULT_BLOCK_SIZE);
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
SimulatedFSDataset.setFactory(conf);
+
conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
}
static void initConfWithRamDisk(Configuration conf) {
@@ -227,6 +228,8 @@ public class TestBalancer {
conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, 1);
conf.setInt(DFS_DATANODE_RAM_DISK_LOW_WATERMARK_BYTES, DEFAULT_RAM_DISK_BLOCK_SIZE);
+
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
}
/* create a file with a length of <code>fileLen</code> */
@@ -1421,6 +1424,7 @@ public class TestBalancer {
}
/**
+
* Test special case. Two replicas belong to same block should not in same node.
* We have 2 nodes.
* We have a block in (DN0,SSD) and (DN1,DISK).
@@ -1436,6 +1440,8 @@ public class TestBalancer {
conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
+
int numOfDatanodes =2;
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(2)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org