You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/10/27 22:47:57 UTC
[46/50] hadoop git commit: HDFS-9305. Delayed heartbeat processing
causes storm of subsequent heartbeats. (Contributed by Arpit Agarwal)
HDFS-9305. Delayed heartbeat processing causes storm of subsequent heartbeats. (Contributed by Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8736eb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8736eb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8736eb9
Branch: refs/heads/HDFS-8966
Commit: d8736eb9ca351b82854601ea3b1fbc3c9fab44e4
Parents: e8aefdf
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Oct 26 15:45:02 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Oct 26 15:54:14 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/datanode/BPServiceActor.java | 4 ++--
.../datanode/TestBpServiceActorScheduler.java | 22 ++++++++++++++++++++
3 files changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8736eb9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 478d48b..e26abcc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2218,6 +2218,9 @@ Release 2.7.2 - UNRELEASED
HDFS-9290. DFSClient#callAppend() is not backward compatible for slightly
older NameNodes. (Tony Wu via kihwal)
+ HDFS-9305. Delayed heartbeat processing causes storm of subsequent
+ heartbeats. (Arpit Agarwal)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8736eb9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 85ea6ae..575e7cc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -538,6 +538,7 @@ class BPServiceActor implements Runnable {
HeartbeatResponse sendHeartBeat(boolean requestBlockReportLease)
throws IOException {
+ scheduler.scheduleNextHeartbeat();
StorageReport[] reports =
dn.getFSDataset().getStorageReports(bpos.getBlockPoolId());
if (LOG.isDebugEnabled()) {
@@ -651,7 +652,6 @@ class BPServiceActor implements Runnable {
//
boolean requestBlockReportLease = (fullBlockReportLeaseId == 0) &&
scheduler.isBlockReportDue(startTime);
- scheduler.scheduleNextHeartbeat();
if (!dn.areHeartbeatsDisabledForTests()) {
resp = sendHeartBeat(requestBlockReportLease);
assert resp != null;
@@ -1064,7 +1064,7 @@ class BPServiceActor implements Runnable {
long scheduleNextHeartbeat() {
// Numerical overflow is possible here and is okay.
- nextHeartbeatTime += heartbeatIntervalMs;
+ nextHeartbeatTime = monotonicNow() + heartbeatIntervalMs;
return nextHeartbeatTime;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8736eb9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
index b9b6512..efdd87c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBpServiceActorScheduler.java
@@ -144,6 +144,28 @@ public class TestBpServiceActorScheduler {
}
}
+
+ /**
+ * Regression test for HDFS-9305.
+ * Delayed processing of a heartbeat can cause a subsequent heartbeat
+ * storm.
+ */
+ @Test
+ public void testScheduleDelayedHeartbeat() {
+ for (final long now : getTimestamps()) {
+ Scheduler scheduler = makeMockScheduler(now);
+ scheduler.scheduleNextHeartbeat();
+ assertFalse(scheduler.isHeartbeatDue(now));
+
+ // Simulate a delayed heartbeat e.g. due to slow processing by NN.
+ scheduler.nextHeartbeatTime = now - (HEARTBEAT_INTERVAL_MS * 10);
+ scheduler.scheduleNextHeartbeat();
+
+ // Ensure that the next heartbeat is not due immediately.
+ assertFalse(scheduler.isHeartbeatDue(now));
+ }
+ }
+
private Scheduler makeMockScheduler(long now) {
LOG.info("Using now = " + now);
Scheduler mockScheduler = spy(new Scheduler(HEARTBEAT_INTERVAL_MS, BLOCK_REPORT_INTERVAL_MS));