You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by va...@apache.org on 2016/10/14 18:03:26 UTC
hadoop git commit: YARN-5599. Publish AM launch command to ATS
(Rohith Sharma K S via Varun Saxena)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 156b92e36 -> e9a58691a
YARN-5599. Publish AM launch command to ATS (Rohith Sharma K S via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9a58691
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9a58691
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9a58691
Branch: refs/heads/branch-2
Commit: e9a58691ab2d48b68af0dcfb64c62b6be033ec89
Parents: 156b92e
Author: Varun Saxena <va...@apache.org>
Authored: Fri Oct 14 23:32:29 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Oct 14 23:32:29 2016 +0530
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 12 -----------
.../src/main/resources/yarn-default.xml | 13 ------------
.../metrics/ApplicationMetricsConstants.java | 3 +++
.../resourcemanager/amlauncher/AMLauncher.java | 21 --------------------
.../metrics/ApplicationCreatedEvent.java | 10 +++++++++-
.../metrics/SystemMetricsPublisher.java | 9 ++++++++-
.../metrics/TestSystemMetricsPublisher.java | 17 ++++++++++++++++
7 files changed, 37 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9a58691/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ba7d046..5716e3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -508,18 +508,6 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
10;
- /**
- * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the
- * command being executed to the RM log if this property is true. Commands
- * may contain sensitive information, such as application or service
- * passwords, making logging the commands a security risk. In cases where
- * the cluster may be running applications with such commands, this property
- * should be set to false. Commands are only logged at the debug level.
- */
- public static final String RM_AMLAUNCHER_LOG_COMMAND =
- RM_PREFIX + "amlauncher.log.command";
- public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false;
-
//RM delegation token related keys
public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
RM_PREFIX + "delegation.key.update-interval";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9a58691/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1d3327a..a1eff52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -299,19 +299,6 @@
</property>
<property>
- <description>
- The resource manager will log all commands being executed to the RM log
- if this property is true. Commands may contain sensitive information,
- such as application or service passwords, making logging the commands a
- security risk. In cases where the cluster may be running applications with
- such commands this property should be set to false. Commands are only
- logged at the debug level.
- </description>
- <name>yarn.resourcemanager.amlauncher.log.command</name>
- <value>false</value>
- </property>
-
- <property>
<description>The class to use as the resource scheduler.</description>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9a58691/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index d06b7cb..1774208 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -98,4 +98,7 @@ public class ApplicationMetricsConstants {
public static final String AM_NODE_LABEL_EXPRESSION =
"YARN_AM_NODE_LABEL_EXPRESSION";
+
+ public static final String AM_CONTAINER_LAUNCH_COMMAND =
+ "YARN_AM_CONTAINER_LAUNCH_COMMAND";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9a58691/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index e7105f9..e513198 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptI
import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
/**
* The launch of the AM itself.
@@ -80,7 +79,6 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
- private final boolean logCommandLine;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@@ -93,9 +91,6 @@ public class AMLauncher implements Runnable {
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
- this.logCommandLine =
- conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND,
- YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND);
}
private void connect() throws IOException {
@@ -192,22 +187,6 @@ public class AMLauncher implements Runnable {
ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
- if (LOG.isDebugEnabled()) {
- StringBuilder message = new StringBuilder("Command to launch container ");
-
- message.append(containerID).append(" : ");
-
- if (logCommandLine) {
- message.append(Joiner.on(",").join(container.getCommands()));
- } else {
- message.append("<REDACTED> -- Set ");
- message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND);
- message.append(" to true to reenable command logging");
- }
-
- LOG.debug(message.toString());
- }
-
// Populate the current queue name in the environment variable.
setupQueueNameEnv(container, applicationMasterContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9a58691/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
index 968a8fd..1adf89f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
@@ -22,6 +22,7 @@ import java.util.Set;
import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.Priority;
public class ApplicationCreatedEvent extends
@@ -39,6 +40,7 @@ public class ApplicationCreatedEvent extends
private String appNodeLabelsExpression;
private String amNodeLabelsExpression;
private final CallerContext callerContext;
+ private ContainerLaunchContext amContainerSpec;
public ApplicationCreatedEvent(ApplicationId appId,
@@ -53,7 +55,8 @@ public class ApplicationCreatedEvent extends
Priority applicationPriority,
String appNodeLabelsExpression,
String amNodeLabelsExpression,
- CallerContext callerContext) {
+ CallerContext callerContext,
+ ContainerLaunchContext amContainerSpec) {
super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId;
this.name = name;
@@ -67,6 +70,7 @@ public class ApplicationCreatedEvent extends
this.appNodeLabelsExpression = appNodeLabelsExpression;
this.amNodeLabelsExpression = amNodeLabelsExpression;
this.callerContext = callerContext;
+ this.amContainerSpec = amContainerSpec;
}
@Override
@@ -121,4 +125,8 @@ public class ApplicationCreatedEvent extends
public CallerContext getCallerContext() {
return callerContext;
}
+
+ public ContainerLaunchContext getAmContainerSpec() {
+ return amContainerSpec;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9a58691/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index b0b976d..250523a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@@ -118,7 +119,8 @@ public class SystemMetricsPublisher extends CompositeService {
appSubmissionContext.getPriority(),
app.getAppNodeLabelExpression(),
app.getAmNodeLabelExpression(),
- app.getCallerContext()));
+ app.getCallerContext(),
+ appSubmissionContext.getAMContainerSpec()));
}
}
@@ -318,6 +320,11 @@ public class SystemMetricsPublisher extends CompositeService {
event.getCallerContext().getSignature());
}
}
+
+ ContainerLaunchContext amContainerSpec = event.getAmContainerSpec();
+ entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND,
+ amContainerSpec.getCommands());
+
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9a58691/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 9ea2baa..914e5b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.Collection;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.Map;
@@ -33,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -118,6 +120,11 @@ public class TestSystemMetricsPublisher {
when(asc.getUnmanagedAM()).thenReturn(false);
when(asc.getPriority()).thenReturn(Priority.newInstance(1));
when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
+ ContainerLaunchContext containerLaunchContext =
+ mock(ContainerLaunchContext.class);
+ when(containerLaunchContext.getCommands())
+ .thenReturn(Collections.singletonList("java -Xmx1024m"));
+ when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
when(app.getApplicationSubmissionContext()).thenReturn(asc);
metricsPublisher.appUpdated(app, 4L);
} else {
@@ -197,6 +204,11 @@ public class TestSystemMetricsPublisher {
Assert.assertEquals("uers1,user2",
entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
+ Assert.assertEquals(
+ app.getApplicationSubmissionContext().getAMContainerSpec()
+ .getCommands(),
+ entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND));
} else {
Assert.assertEquals(
"",
@@ -492,6 +504,11 @@ public class TestSystemMetricsPublisher {
when(asc.getUnmanagedAM()).thenReturn(false);
when(asc.getPriority()).thenReturn(Priority.newInstance(10));
when(asc.getNodeLabelExpression()).thenReturn("high-cpu");
+ ContainerLaunchContext containerLaunchContext =
+ mock(ContainerLaunchContext.class);
+ when(containerLaunchContext.getCommands())
+ .thenReturn(Collections.singletonList("java -Xmx1024m"));
+ when(asc.getAMContainerSpec()).thenReturn(containerLaunchContext);
when(app.getApplicationSubmissionContext()).thenReturn(asc);
when(app.getAppNodeLabelExpression()).thenCallRealMethod();
ResourceRequest amReq = mock(ResourceRequest.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org