You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2019/04/18 16:30:03 UTC
[hadoop] branch trunk updated: YARN-6695. Fixed NPE in publishing
appFinished events to ATSv2. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new df76cdc YARN-6695. Fixed NPE in publishing appFinished events to ATSv2. Contributed by Prabhu Joseph
df76cdc is described below
commit df76cdc8959c51b71704ab5c38335f745a6f35d8
Author: Eric Yang <ey...@apache.org>
AuthorDate: Thu Apr 18 12:29:37 2019 -0400
YARN-6695. Fixed NPE in publishing appFinished events to ATSv2.
Contributed by Prabhu Joseph
---
.../metrics/TimelineServiceV2Publisher.java | 13 ++++--
.../metrics/TestSystemMetricsPublisherForV2.java | 47 ++++++++++++++++++++++
.../src/site/markdown/TimelineServiceV2.md | 8 ----
3 files changed, 56 insertions(+), 12 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 1438e25..eeeed48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -473,10 +473,15 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
}
TimelineCollector timelineCollector =
rmTimelineCollectorManager.get(appId);
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
- timelineCollector.putEntities(entities,
- UserGroupInformation.getCurrentUser());
+ if (timelineCollector != null) {
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ timelineCollector.putEntities(entities,
+ UserGroupInformation.getCurrentUser());
+ } else {
+ LOG.debug("Cannot find active collector while publishing entity "
+ + entity);
+ }
} catch (IOException e) {
LOG.error("Error when publishing entity " + entity);
LOG.debug("Error when publishing entity {}", entity, e);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 5e1a224..76e8f0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -28,12 +28,17 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -293,6 +298,48 @@ public class TestSystemMetricsPublisherForV2 {
TimelineServiceHelper.invertLong(containerId.getContainerId()));
}
+ @Test(timeout = 10000)
+ public void testPutEntityWhenNoCollector() throws Exception {
+ // Validating the logs as DrainDispatcher won't throw exception
+ class TestAppender extends AppenderSkeleton {
+ private final List<LoggingEvent> log = new ArrayList<>();
+
+ @Override
+ public boolean requiresLayout() {
+ return false;
+ }
+
+ @Override
+ protected void append(final LoggingEvent loggingEvent) {
+ log.add(loggingEvent);
+ }
+
+ @Override
+ public void close() {
+ }
+
+ public List<LoggingEvent> getLog() {
+ return new ArrayList<>(log);
+ }
+ }
+
+ TestAppender appender = new TestAppender();
+ final Logger logger = Logger.getRootLogger();
+ logger.addAppender(appender);
+
+ try {
+ RMApp app = createRMApp(ApplicationId.newInstance(0, 1));
+ metricsPublisher.appCreated(app, app.getStartTime());
+ dispatcher.await();
+ for (LoggingEvent event : appender.getLog()) {
+ assertFalse("Dispatcher Crashed",
+ event.getRenderedMessage().contains("Error in dispatcher thread"));
+ }
+ } finally {
+ logger.removeAppender(appender);
+ }
+ }
+
private RMApp createAppAndRegister(ApplicationId appId) {
RMApp app = createRMApp(appId);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
index cf98c57..b75a9d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
@@ -330,14 +330,6 @@ Following are the basic configurations to start Timeline service v.2:
<name>yarn.system-metrics-publisher.enabled</name>
<value>true</value>
</property>
-
-<property>
- <description>The setting that controls whether yarn container events are
- published to the timeline service or not by RM. This configuration setting
- is for ATS V2.</description>
- <name>yarn.rm.system-metrics-publisher.emit-container-events</name>
- <value>true</value>
-</property>
```
If using an aux services manifest instead of setting aux services through the Configuration, ensure that the manifest services array includes the timeline\_collector service as follows:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org