You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2019/02/23 02:33:51 UTC
[hadoop] branch branch-3.1 updated: YARN-9317. Avoid repeated
YarnConfiguration#timelineServiceV2Enabled check. Contributed by Prabhu
Joseph
This is an automated email from the ASF dual-hosted git repository.
bibinchundatt pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 6d1cf7b YARN-9317. Avoid repeated YarnConfiguration#timelineServiceV2Enabled check. Contributed by Prabhu Joseph
6d1cf7b is described below
commit 6d1cf7b3959d645fdbbd0eb0c702feb3d32a7a69
Author: bibinchundatt <bi...@apache.org>
AuthorDate: Sat Feb 23 08:03:28 2019 +0530
YARN-9317. Avoid repeated YarnConfiguration#timelineServiceV2Enabled check. Contributed by Prabhu Joseph
---
.../yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 5 ++++-
.../containermanager/ContainerManagerImpl.java | 15 +++++++++------
.../server/resourcemanager/ApplicationMasterService.java | 5 ++++-
.../yarn/server/resourcemanager/ClientRMService.java | 5 ++++-
.../yarn/server/resourcemanager/DefaultAMSProcessor.java | 6 ++++--
.../hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 ++++-
.../server/resourcemanager/ResourceTrackerService.java | 9 +++++----
.../server/resourcemanager/amlauncher/AMLauncher.java | 5 ++++-
8 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index e79ab47..8eb89dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -153,6 +153,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private NMNodeLabelsHandler nodeLabelsHandler;
private final NodeLabelsProvider nodeLabelsProvider;
+ private boolean timelineServiceV2Enabled;
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
@@ -240,6 +241,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
this.logAggregationEnabled =
conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED);
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
}
@Override
@@ -1177,7 +1180,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
newResource.toString());
}
}
- if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) {
+ if (timelineServiceV2Enabled) {
updateTimelineCollectorData(response);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 41b7418..3c63ec6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -227,6 +227,7 @@ public class ContainerManagerImpl extends CompositeService implements
// NM metrics publisher is set only if the timeline service v.2 is enabled
private NMTimelinePublisher nmMetricsPublisher;
+ private boolean timelineServiceV2Enabled;
public ContainerManagerImpl(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
@@ -263,11 +264,13 @@ public class ContainerManagerImpl extends CompositeService implements
// initialize the metrics publisher if the timeline service v.2 is enabled
// and the system publisher is enabled
Configuration conf = context.getConf();
- if (YarnConfiguration.timelineServiceV2Enabled(conf) &&
- YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
- LOG.info("YARN system metrics publishing service is enabled");
- nmMetricsPublisher = createNMTimelinePublisher(context);
- context.setNMTimelinePublisher(nmMetricsPublisher);
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
+ LOG.info("YARN system metrics publishing service is enabled");
+ nmMetricsPublisher = createNMTimelinePublisher(context);
+ context.setNMTimelinePublisher(nmMetricsPublisher);
+ }
+ this.timelineServiceV2Enabled = true;
}
this.containersMonitor = createContainersMonitor(exec);
addService(this.containersMonitor);
@@ -1184,7 +1187,7 @@ public class ContainerManagerImpl extends CompositeService implements
private FlowContext getFlowContext(ContainerLaunchContext launchContext,
ApplicationId applicationID) {
FlowContext flowContext = null;
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ if (timelineServiceV2Enabled) {
String flowName = launchContext.getEnvironment()
.get(TimelineUtils.FLOW_NAME_TAG_PREFIX);
String flowVersion = launchContext.getEnvironment()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index ae28879..49bc21c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -96,6 +96,7 @@ public class ApplicationMasterService extends AbstractService implements
new ConcurrentHashMap<ApplicationAttemptId, AllocateResponseLock>();
protected final RMContext rmContext;
private final AMSProcessingChain amsProcessingChain;
+ private boolean timelineServiceV2Enabled;
public ApplicationMasterService(RMContext rmContext,
YarnScheduler scheduler) {
@@ -213,6 +214,8 @@ public class ApplicationMasterService extends AbstractService implements
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
server.getListenerAddress());
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
super.serviceStart();
}
@@ -303,7 +306,7 @@ public class ApplicationMasterService extends AbstractService implements
rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
// Remove collector address when app get finished.
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ if (timelineServiceV2Enabled) {
((RMAppImpl) rmApp).removeCollectorData();
}
// checking whether the app exits in RMStateStore at first not to throw
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 2ae1031..379251e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -222,6 +222,7 @@ public class ClientRMService extends AbstractService implements
RMAppState.ACCEPTED, RMAppState.RUNNING);
private ResourceProfilesManager resourceProfilesManager;
+ private boolean timelineServiceV2Enabled;
public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
@@ -292,6 +293,8 @@ public class ClientRMService extends AbstractService implements
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
server.getListenerAddress());
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
super.serviceStart();
}
@@ -571,7 +574,7 @@ public class ClientRMService extends AbstractService implements
throw RPCUtil.getRemoteException(ie);
}
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ if (timelineServiceV2Enabled) {
// Sanity check for flow run
String value = null;
try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 5168bd3..d3ce241 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -107,12 +107,15 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
private RMContext rmContext;
private ResourceProfilesManager resourceProfilesManager;
+ private boolean timelineServiceV2Enabled;
@Override
public void init(ApplicationMasterServiceContext amsContext,
ApplicationMasterServiceProcessor nextProcessor) {
this.rmContext = (RMContext)amsContext;
this.resourceProfilesManager = rmContext.getResourceProfilesManager();
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(rmContext.getYarnConfiguration());
}
@Override
@@ -318,8 +321,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
response.setNumClusterNodes(getScheduler().getNumClusterNodes());
// add collector address for this application
- if (YarnConfiguration.timelineServiceV2Enabled(
- getRmContext().getYarnConfiguration())) {
+ if (timelineServiceV2Enabled) {
CollectorInfo collectorInfo = app.getCollectorInfo();
if (collectorInfo != null) {
response.setCollectorInfo(collectorInfo);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 9369eb2..7f9f51c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -94,6 +94,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private final ApplicationACLsManager applicationACLsManager;
private Configuration conf;
private YarnAuthorizationProvider authorizer;
+ private boolean timelineServiceV2Enabled;
public RMAppManager(RMContext context,
YarnScheduler scheduler, ApplicationMasterService masterService,
@@ -114,6 +115,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
}
this.authorizer = YarnAuthorizationProvider.getInstance(conf);
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
}
/**
@@ -451,7 +454,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
throw new YarnException(message);
}
- if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ if (timelineServiceV2Enabled) {
// Start timeline collector for the submitted app
application.startTimelineCollector();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index e997192..81be628 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -124,6 +124,7 @@ public class ResourceTrackerService extends AbstractService implements
private DynamicResourceConfiguration drConf;
private final AtomicLong timelineCollectorVersion = new AtomicLong(0);
+ private boolean timelineServiceV2Enabled;
public ResourceTrackerService(RMContext rmContext,
NodesListManager nodesListManager,
@@ -170,6 +171,8 @@ public class ResourceTrackerService extends AbstractService implements
minimumNodeManagerVersion = conf.get(
YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);
+ timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
if (YarnConfiguration.areNodeLabelsEnabled(conf)) {
isDistributedNodeLabelsConf =
@@ -578,9 +581,7 @@ public class ResourceTrackerService extends AbstractService implements
NodeAction.SHUTDOWN, message);
}
- boolean timelineV2Enabled =
- YarnConfiguration.timelineServiceV2Enabled(getConfig());
- if (timelineV2Enabled) {
+ if (timelineServiceV2Enabled) {
// Check & update collectors info from request.
updateAppCollectorsMap(request);
}
@@ -600,7 +601,7 @@ public class ResourceTrackerService extends AbstractService implements
nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials);
}
- if (timelineV2Enabled) {
+ if (timelineServiceV2Enabled) {
// Return collectors' map that NM needs to know
setAppCollectorsMapToResponse(rmNode.getRunningApps(),
nodeHeartBeatResponse);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 0bedb52..3ec329f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -81,6 +81,7 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
+ private boolean timelineServiceV2Enabled;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@@ -93,6 +94,8 @@ public class AMLauncher implements Runnable {
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
}
private void connect() throws IOException {
@@ -239,7 +242,7 @@ public class AMLauncher implements Runnable {
}
private void setFlowContext(ContainerLaunchContext container) {
- if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ if (timelineServiceV2Enabled) {
Map<String, String> environment = container.getEnvironment();
ApplicationId applicationId =
application.getAppAttemptId().getApplicationId();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org