You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/08/20 00:01:33 UTC
[10/30] hadoop git commit: HDFS-8278. When computing max-size-to-move
in Balancer, count only the storage with remaining >= default block size.
HDFS-8278. When computing max-size-to-move in Balancer, count only the storage with remaining >= default block size.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51a00964
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51a00964
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51a00964
Branch: refs/heads/YARN-1197
Commit: 51a00964da0e399718d1cec25ff692a32d7642b7
Parents: eee4d71
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Mon Aug 17 17:55:25 2015 -0700
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Mon Aug 17 17:55:25 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/balancer/Balancer.java | 16 +++++++++++-----
2 files changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51a00964/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7d05bc2..d1b04dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -803,6 +803,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8880. NameNode metrics logging. (Arpit Agarwal)
+ HDFS-8278. When computing max-size-to-move in Balancer, count only the
+ storage with remaining >= default block size. (szetszwo)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51a00964/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 6fc024e..e92ed81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -191,6 +191,7 @@ public class Balancer {
private final boolean runDuringUpgrade;
private final double threshold;
private final long maxSizeToMove;
+ private final long defaultBlockSize;
// all data node lists
private final Collection<Source> overUtilized = new LinkedList<Source>();
@@ -269,6 +270,9 @@ public class Balancer {
this.maxSizeToMove = getLong(conf,
DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_KEY,
DFSConfigKeys.DFS_BALANCER_MAX_SIZE_TO_MOVE_DEFAULT);
+ this.defaultBlockSize = getLong(conf,
+ DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
+ DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT);
}
private static long getCapacity(DatanodeStorageReport report, StorageType t) {
@@ -281,11 +285,13 @@ public class Balancer {
return capacity;
}
- private static long getRemaining(DatanodeStorageReport report, StorageType t) {
+ private long getRemaining(DatanodeStorageReport report, StorageType t) {
long remaining = 0L;
for(StorageReport r : report.getStorageReports()) {
if (r.getStorage().getStorageType() == t) {
- remaining += r.getRemaining();
+ if (r.getRemaining() >= defaultBlockSize) {
+ remaining += r.getRemaining();
+ }
}
}
return remaining;
@@ -322,7 +328,7 @@ public class Balancer {
final double utilizationDiff = utilization - policy.getAvgUtilization(t);
final double thresholdDiff = Math.abs(utilizationDiff) - threshold;
final long maxSize2Move = computeMaxSize2Move(capacity,
- getRemaining(r, t), utilizationDiff, threshold, maxSizeToMove);
+ getRemaining(r, t), utilizationDiff, maxSizeToMove);
final StorageGroup g;
if (utilizationDiff > 0) {
@@ -359,8 +365,8 @@ public class Balancer {
}
private static long computeMaxSize2Move(final long capacity, final long remaining,
- final double utilizationDiff, final double threshold, final long max) {
- final double diff = Math.min(threshold, Math.abs(utilizationDiff));
+ final double utilizationDiff, final long max) {
+ final double diff = Math.abs(utilizationDiff);
long maxSizeToMove = percentage2bytes(diff, capacity);
if (utilizationDiff < 0) {
maxSizeToMove = Math.min(remaining, maxSizeToMove);