You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2021/03/24 05:48:09 UTC
[hadoop] branch trunk updated: HDFS-15911 : Provide blocks moved
count in Balancer iteration result (#2794)
This is an automated email from the ASF dual-hosted git repository.
liuml07 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4b4ccce HDFS-15911 : Provide blocks moved count in Balancer iteration result (#2794)
4b4ccce is described below
commit 4b4ccce02f591f63dff7db346de39c8d996e8f1d
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Wed Mar 24 11:17:45 2021 +0530
HDFS-15911 : Provide blocks moved count in Balancer iteration result (#2794)
Contributed by Viraj Jasani.
Signed-off-by: Mingliang Liu <li...@apache.org>
Signed-off-by: Ayush Saxena <ay...@apache.org>
---
.../hadoop/hdfs/server/balancer/Balancer.java | 28 ++++++++++++++++++----
.../hadoop/hdfs/server/balancer/TestBalancer.java | 8 +++++--
2 files changed, 30 insertions(+), 6 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 0024ba5..33650ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -38,6 +38,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.slf4j.Logger;
@@ -638,13 +639,15 @@ public class Balancer {
private final long bytesLeftToMove;
private final long bytesBeingMoved;
private final long bytesAlreadyMoved;
+ private final long blocksMoved;
Result(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved,
- long bytesAlreadyMoved) {
+ long bytesAlreadyMoved, long blocksMoved) {
this.exitStatus = exitStatus;
this.bytesLeftToMove = bytesLeftToMove;
this.bytesBeingMoved = bytesBeingMoved;
this.bytesAlreadyMoved = bytesAlreadyMoved;
+ this.blocksMoved = blocksMoved;
}
public ExitStatus getExitStatus() {
@@ -663,23 +666,40 @@ public class Balancer {
return bytesAlreadyMoved;
}
+ public long getBlocksMoved() {
+ return blocksMoved;
+ }
+
void print(int iteration, NameNodeConnector nnc, PrintStream out) {
- out.printf("%-24s %10d %19s %18s %17s %s%n",
+ out.printf("%-24s %10d %19s %18s %17s %17s %s%n",
DateFormat.getDateTimeInstance().format(new Date()), iteration,
StringUtils.byteDesc(bytesAlreadyMoved),
StringUtils.byteDesc(bytesLeftToMove),
StringUtils.byteDesc(bytesBeingMoved),
+ blocksMoved,
nnc.getNameNodeUri());
}
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this)
+ .append("exitStatus", exitStatus)
+ .append("bytesLeftToMove", bytesLeftToMove)
+ .append("bytesBeingMoved", bytesBeingMoved)
+ .append("bytesAlreadyMoved", bytesAlreadyMoved)
+ .append("blocksMoved", blocksMoved)
+ .toString();
+ }
}
Result newResult(ExitStatus exitStatus, long bytesLeftToMove, long bytesBeingMoved) {
return new Result(exitStatus, bytesLeftToMove, bytesBeingMoved,
- dispatcher.getBytesMoved());
+ dispatcher.getBytesMoved(), dispatcher.getBblocksMoved());
}
Result newResult(ExitStatus exitStatus) {
- return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved());
+ return new Result(exitStatus, -1, -1, dispatcher.getBytesMoved(),
+ dispatcher.getBblocksMoved());
}
/** Run an iteration for all datanodes. */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 343faf6..f59743f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -1658,6 +1658,7 @@ public class TestBalancer {
// a block is moved unexpectedly, IN_PROGRESS will be reported.
assertEquals("We expect ExitStatus.NO_MOVE_PROGRESS to be reported.",
ExitStatus.NO_MOVE_PROGRESS, r.getExitStatus());
+ assertEquals(0, r.getBlocksMoved());
}
} finally {
for (NameNodeConnector nnc : connectors) {
@@ -2309,8 +2310,11 @@ public class TestBalancer {
// Hence, overall total blocks moved by HDFS balancer would be either of these 2 options:
// a) 2 blocks of total size (100B + 100B)
// b) 3 blocks of total size (50B + 100B + 100B)
- assertTrue(balancerResult.getBytesAlreadyMoved() == 200
- || balancerResult.getBytesAlreadyMoved() == 250);
+ assertTrue("BalancerResult is not as expected. " + balancerResult,
+ (balancerResult.getBytesAlreadyMoved() == 200
+ && balancerResult.getBlocksMoved() == 2)
+ || (balancerResult.getBytesAlreadyMoved() == 250
+ && balancerResult.getBlocksMoved() == 3));
// 100% and 95% used nodes will be balanced, so top used will be 900
assertEquals(900, maxUsage);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org