You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2019/02/23 02:30:14 UTC
[hadoop] branch branch-3.2 updated: YARN-9317. Avoid repeated
YarnConfiguration#timelineServiceV2Enabled check. Contributed by Prabhu
Joseph
This is an automated email from the ASF dual-hosted git repository.
bibinchundatt pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 3e1bd53 YARN-9317. Avoid repeated YarnConfiguration#timelineServiceV2Enabled check. Contributed by Prabhu Joseph
3e1bd53 is described below
commit 3e1bd53a376eec7af2fd2e764ccdae33e934f78d
Author: bibinchundatt <bi...@apache.org>
AuthorDate: Sat Feb 23 07:59:51 2019 +0530
YARN-9317. Avoid repeated YarnConfiguration#timelineServiceV2Enabled check. Contributed by Prabhu Joseph
---
.../yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 5 ++++-
.../containermanager/ContainerManagerImpl.java | 15 +++++++++------
.../server/resourcemanager/ApplicationMasterService.java | 5 ++++-
.../yarn/server/resourcemanager/ClientRMService.java | 5 ++++-
.../yarn/server/resourcemanager/DefaultAMSProcessor.java | 6 ++++--
.../hadoop/yarn/server/resourcemanager/RMAppManager.java | 5 ++++-
.../server/resourcemanager/ResourceTrackerService.java | 9 +++++----
.../server/resourcemanager/amlauncher/AMLauncher.java | 5 ++++-
8 files changed, 38 insertions(+), 17 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index b570625..08e7372 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -158,6 +158,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private NMNodeAttributesHandler nodeAttributesHandler;
private NodeLabelsProvider nodeLabelsProvider;
private NodeAttributesProvider nodeAttributesProvider;
+ private boolean timelineServiceV2Enabled;
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
@@ -252,6 +253,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
this.logAggregationEnabled =
conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED);
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
}
@Override
@@ -1406,7 +1409,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
newResource.toString());
}
}
- if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) {
+ if (timelineServiceV2Enabled) {
updateTimelineCollectorData(response);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 01d70af..c827fcf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -227,6 +227,7 @@ public class ContainerManagerImpl extends CompositeService implements
// NM metrics publisher is set only if the timeline service v.2 is enabled
private NMTimelinePublisher nmMetricsPublisher;
+ private boolean timelineServiceV2Enabled;
public ContainerManagerImpl(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
@@ -263,11 +264,13 @@ public class ContainerManagerImpl extends CompositeService implements
// initialize the metrics publisher if the timeline service v.2 is enabled
// and the system publisher is enabled
Configuration conf = context.getConf();
- if (YarnConfiguration.timelineServiceV2Enabled(conf) &&
- YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
- LOG.info("YARN system metrics publishing service is enabled");
- nmMetricsPublisher = createNMTimelinePublisher(context);
- context.setNMTimelinePublisher(nmMetricsPublisher);
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
+ LOG.info("YARN system metrics publishing service is enabled");
+ nmMetricsPublisher = createNMTimelinePublisher(context);
+ context.setNMTimelinePublisher(nmMetricsPublisher);
+ }
+ this.timelineServiceV2Enabled = true;
}
this.containersMonitor = createContainersMonitor(exec);
addService(this.containersMonitor);
@@ -1184,7 +1187,7 @@ public class ContainerManagerImpl extends CompositeService implements
private FlowContext getFlowContext(ContainerLaunchContext launchContext,
ApplicationId applicationID) {
FlowContext flowContext = null;
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ if (timelineServiceV2Enabled) {
String flowName = launchContext.getEnvironment()
.get(TimelineUtils.FLOW_NAME_TAG_PREFIX);
String flowVersion = launchContext.getEnvironment()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 70b7498..d9527d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -95,6 +95,7 @@ public class ApplicationMasterService extends AbstractService implements
new ConcurrentHashMap<ApplicationAttemptId, AllocateResponseLock>();
protected final RMContext rmContext;
private final AMSProcessingChain amsProcessingChain;
+ private boolean timelineServiceV2Enabled;
public ApplicationMasterService(RMContext rmContext,
YarnScheduler scheduler) {
@@ -212,6 +213,8 @@ public class ApplicationMasterService extends AbstractService implements
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
server.getListenerAddress());
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
super.serviceStart();
}
@@ -302,7 +305,7 @@ public class ApplicationMasterService extends AbstractService implements
rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
// Remove collector address when app get finished.
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ if (timelineServiceV2Enabled) {
((RMAppImpl) rmApp).removeCollectorData();
}
// checking whether the app exits in RMStateStore at first not to throw
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 062f39a..e2765a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -236,6 +236,7 @@ public class ClientRMService extends AbstractService implements
RMAppState.ACCEPTED, RMAppState.RUNNING);
private ResourceProfilesManager resourceProfilesManager;
+ private boolean timelineServiceV2Enabled;
public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
@@ -306,6 +307,8 @@ public class ClientRMService extends AbstractService implements
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
server.getListenerAddress());
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
super.serviceStart();
}
@@ -585,7 +588,7 @@ public class ClientRMService extends AbstractService implements
throw RPCUtil.getRemoteException(ie);
}
- if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ if (timelineServiceV2Enabled) {
// Sanity check for flow run
String value = null;
try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index b2c5ef3..9e05eda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -115,12 +115,15 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
private RMContext rmContext;
private ResourceProfilesManager resourceProfilesManager;
+ private boolean timelineServiceV2Enabled;
@Override
public void init(ApplicationMasterServiceContext amsContext,
ApplicationMasterServiceProcessor nextProcessor) {
this.rmContext = (RMContext)amsContext;
this.resourceProfilesManager = rmContext.getResourceProfilesManager();
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(rmContext.getYarnConfiguration());
}
@Override
@@ -326,8 +329,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
response.setNumClusterNodes(getScheduler().getNumClusterNodes());
// add collector address for this application
- if (YarnConfiguration.timelineServiceV2Enabled(
- getRmContext().getYarnConfiguration())) {
+ if (timelineServiceV2Enabled) {
CollectorInfo collectorInfo = app.getCollectorInfo();
if (collectorInfo != null) {
response.setCollectorInfo(collectorInfo);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 3c5436c..0679b0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -95,6 +95,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private final ApplicationACLsManager applicationACLsManager;
private Configuration conf;
private YarnAuthorizationProvider authorizer;
+ private boolean timelineServiceV2Enabled;
public RMAppManager(RMContext context,
YarnScheduler scheduler, ApplicationMasterService masterService,
@@ -115,6 +116,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
}
this.authorizer = YarnAuthorizationProvider.getInstance(conf);
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
}
/**
@@ -493,7 +496,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
throw new YarnException(message);
}
- if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ if (timelineServiceV2Enabled) {
// Start timeline collector for the submitted app
application.startTimelineCollector();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index f3d8eb2..9cff85b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -127,6 +127,7 @@ public class ResourceTrackerService extends AbstractService implements
private DynamicResourceConfiguration drConf;
private final AtomicLong timelineCollectorVersion = new AtomicLong(0);
+ private boolean timelineServiceV2Enabled;
public ResourceTrackerService(RMContext rmContext,
NodesListManager nodesListManager,
@@ -173,6 +174,8 @@ public class ResourceTrackerService extends AbstractService implements
minimumNodeManagerVersion = conf.get(
YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);
+ timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
if (YarnConfiguration.areNodeLabelsEnabled(conf)) {
isDistributedNodeLabelsConf =
@@ -600,9 +603,7 @@ public class ResourceTrackerService extends AbstractService implements
NodeAction.SHUTDOWN, message);
}
- boolean timelineV2Enabled =
- YarnConfiguration.timelineServiceV2Enabled(getConfig());
- if (timelineV2Enabled) {
+ if (timelineServiceV2Enabled) {
// Check & update collectors info from request.
updateAppCollectorsMap(request);
}
@@ -622,7 +623,7 @@ public class ResourceTrackerService extends AbstractService implements
nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials);
}
- if (timelineV2Enabled) {
+ if (timelineServiceV2Enabled) {
// Return collectors' map that NM needs to know
setAppCollectorsMapToResponse(rmNode.getRunningApps(),
nodeHeartBeatResponse);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 0bedb52..3ec329f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -81,6 +81,7 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
+ private boolean timelineServiceV2Enabled;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@@ -93,6 +94,8 @@ public class AMLauncher implements Runnable {
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
+ this.timelineServiceV2Enabled = YarnConfiguration.
+ timelineServiceV2Enabled(conf);
}
private void connect() throws IOException {
@@ -239,7 +242,7 @@ public class AMLauncher implements Runnable {
}
private void setFlowContext(ContainerLaunchContext container) {
- if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ if (timelineServiceV2Enabled) {
Map<String, String> environment = container.getEnvironment();
ApplicationId applicationId =
application.getAppAttemptId().getApplicationId();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org