You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2020/02/27 08:47:51 UTC
[hadoop] branch trunk updated: HDFS-15167. Block Report Interval
shouldn't be reset apart from first Block Report. Contributed by Ayush
Saxena.
This is an automated email from the ASF dual-hosted git repository.
surendralilhore pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2059f25 HDFS-15167. Block Report Interval shouldn't be reset apart from first Block Report. Contributed by Ayush Saxena.
2059f25 is described below
commit 2059f255d39510cddbb954b21ed4871a737bd2dc
Author: Surendra Singh Lilhore <su...@apache.org>
AuthorDate: Thu Feb 27 14:08:42 2020 +0530
HDFS-15167. Block Report Interval shouldn't be reset apart from first Block Report. Contributed by Ayush Saxena.
---
.../hdfs/server/datanode/BPOfferService.java | 2 +-
.../hdfs/server/datanode/BPServiceActor.java | 30 +++++++++++++++++-----
.../datanode/TestBpServiceActorScheduler.java | 4 +--
3 files changed, 26 insertions(+), 10 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index df793c8..3199e53 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -496,7 +496,7 @@ class BPOfferService {
*/
void scheduleBlockReport(long delay) {
for (BPServiceActor actor : bpServices) {
- actor.getScheduler().scheduleBlockReport(delay);
+ actor.getScheduler().scheduleBlockReport(delay, false);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index f4403e8..43e9745 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -828,7 +828,7 @@ class BPServiceActor implements Runnable {
fullBlockReportLeaseId = 0;
// random short delay - helps scatter the BR from all DNs
- scheduler.scheduleBlockReport(dnConf.initialBlockReportDelayMs);
+ scheduler.scheduleBlockReport(dnConf.initialBlockReportDelayMs, true);
}
@@ -1217,14 +1217,19 @@ class BPServiceActor implements Runnable {
void forceFullBlockReportNow() {
forceFullBlockReport.set(true);
- resetBlockReportTime = true;
}
/**
* This methods arranges for the data node to send the block report at
* the next heartbeat.
+ * @param delay specifies the maximum amount of random delay(in
+ * milliseconds) in sending the block report. A value of 0
+ * or less makes the BR to go right away without any delay.
+ * @param isRegistration if true, resets the future BRs for randomness,
+ * post first BR to avoid regular BRs from all DN's
+ * coming at one time.
*/
- long scheduleBlockReport(long delay) {
+ long scheduleBlockReport(long delay, boolean isRegistration) {
if (delay > 0) { // send BR after random delay
// Numerical overflow is possible here and is okay.
nextBlockReportTime =
@@ -1232,7 +1237,9 @@ class BPServiceActor implements Runnable {
} else { // send at next heartbeat
nextBlockReportTime = monotonicNow();
}
- resetBlockReportTime = true; // reset future BRs for randomness
+ resetBlockReportTime = isRegistration; // reset future BRs for
+ // randomness, post first block report to avoid regular BRs from all
+ // DN's coming at one time.
return nextBlockReportTime;
}
@@ -1257,9 +1264,18 @@ class BPServiceActor implements Runnable {
* 2) unexpected like 21:35:43, next report should be at 2:20:14
* on the next day.
*/
- nextBlockReportTime +=
- (((monotonicNow() - nextBlockReportTime + blockReportIntervalMs) /
- blockReportIntervalMs)) * blockReportIntervalMs;
+ long factor =
+ (monotonicNow() - nextBlockReportTime + blockReportIntervalMs)
+ / blockReportIntervalMs;
+ if (factor != 0) {
+ nextBlockReportTime += factor * blockReportIntervalMs;
+ } else {
+ // If the difference between the present time and the scheduled
+ // time is very less, the factor can be 0, so in that case, we can
+ // ignore that negligible time, spent while sending the BRss and
+ // schedule the next BR after the blockReportInterval.
+ nextBlockReportTime += blockReportIntervalMs;
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
index 438be89..36b6c69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
@@ -68,7 +68,7 @@ public class TestBpServiceActorScheduler {
public void testScheduleBlockReportImmediate() {
for (final long now : getTimestamps()) {
Scheduler scheduler = makeMockScheduler(now);
- scheduler.scheduleBlockReport(0);
+ scheduler.scheduleBlockReport(0, true);
assertTrue(scheduler.resetBlockReportTime);
assertThat(scheduler.nextBlockReportTime, is(now));
}
@@ -79,7 +79,7 @@ public class TestBpServiceActorScheduler {
for (final long now : getTimestamps()) {
Scheduler scheduler = makeMockScheduler(now);
final long delayMs = 10;
- scheduler.scheduleBlockReport(delayMs);
+ scheduler.scheduleBlockReport(delayMs, true);
assertTrue(scheduler.resetBlockReportTime);
assertTrue(scheduler.nextBlockReportTime - now >= 0);
assertTrue(scheduler.nextBlockReportTime - (now + delayMs) < 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org