You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pb...@apache.org on 2021/04/27 12:02:35 UTC
[hadoop] branch trunk updated: YARN-10739.
GenericEventHandler.printEventQueueDetails causes RM recovery to take too
much time. Contributed by Qi Zhu.
This is an automated email from the ASF dual-hosted git repository.
pbacsko pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a967ab0 YARN-10739. GenericEventHandler.printEventQueueDetails causes RM recovery to take too much time. Contributed by Qi Zhu.
a967ab0 is described below
commit a967ab06f2e802f30ed34eb4ceb2742e4df8be52
Author: Peter Bacsko <pb...@cloudera.com>
AuthorDate: Tue Apr 27 14:01:53 2021 +0200
YARN-10739. GenericEventHandler.printEventQueueDetails causes RM recovery to take too much time. Contributed by Qi Zhu.
---
.../org/apache/hadoop/yarn/event/AsyncDispatcher.java | 18 +++++++++++++++++-
.../apache/hadoop/yarn/event/TestAsyncDispatcher.java | 2 --
2 files changed, 17 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 667515d..ba6bb43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -25,7 +25,11 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.yarn.metrics.EventTypeMetrics;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.MonotonicClock;
@@ -93,6 +97,8 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
private Clock clock = new MonotonicClock();
+ private ThreadPoolExecutor printEventDetailsExecutor;
+
/**
* The thread name for dispatcher.
*/
@@ -179,6 +185,15 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD,
YarnConfiguration.
DEFAULT_YARN_DISPATCHER_PRINT_EVENTS_INFO_THRESHOLD);
+
+ ThreadFactory threadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("PrintEventDetailsThread #%d")
+ .build();
+ // Thread pool for async print event details,
+ // to prevent wasting too much time for RM.
+ printEventDetailsExecutor = new ThreadPoolExecutor(
+ 1, 5, 10, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(), threadFactory);
}
@Override
@@ -222,6 +237,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
LOG.warn("Interrupted Exception while stopping", ie);
}
}
+ printEventDetailsExecutor.shutdownNow();
// stop all the components
super.serviceStop();
@@ -319,7 +335,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
if (qSize != 0 && qSize % detailsInterval == 0
&& lastEventDetailsQueueSizeLogged != qSize) {
lastEventDetailsQueueSizeLogged = qSize;
- printEventQueueDetails();
+ printEventDetailsExecutor.submit(this::printEventQueueDetails);
printTrigger = true;
}
int remCapacity = eventQueue.remainingCapacity();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index 7d2572a..8b2dfa0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -187,8 +187,6 @@ public class TestAsyncDispatcher {
when(event.getType()).thenReturn(TestEnum.TestEventType);
dispatcher.getEventHandler().handle(event);
}
- verify(log, atLeastOnce()).info("Event type: TestEventType, " +
- "Event record counter: 5000");
Thread.sleep(2000);
//Make sure more than one event to take
verify(log, atLeastOnce()).
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org