You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2017/08/31 21:30:07 UTC
[1/3] hadoop git commit: YARN-6890. Not display killApp button on UI
if UI is unsecured but cluster is secured. Contributed by Junping Du
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8.2 c93e81f8e -> 8dba358a2
YARN-6890. Not display killApp button on UI if UI is unsecured but cluster is secured. Contributed by Junping Du
(cherry picked from commit bc87a293d91b6ca4f734300f8d91203393f2027a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3e1bff3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3e1bff3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3e1bff3
Branch: refs/heads/branch-2.8.2
Commit: e3e1bff301f054c72465f652b5a992d22c851038
Parents: c93e81f
Author: Jian He <ji...@apache.org>
Authored: Tue Aug 8 11:09:38 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Thu Aug 31 14:28:51 2017 -0700
----------------------------------------------------------------------
.../hadoop/fs/CommonConfigurationKeysPublic.java | 3 +++
.../apache/hadoop/yarn/server/webapp/AppBlock.java | 14 +++++++++++++-
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3e1bff3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index e127350..5b20d96 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -565,6 +565,9 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_SECURITY_DNS_NAMESERVER_KEY =
"hadoop.security.dns.nameserver";
+ public static final String HADOOP_HTTP_AUTHENTICATION_TYPE =
+ "hadoop.http.authentication.type";
+
@Deprecated
/** Only used by HttpServer. */
public static final String HADOOP_SSL_ENABLED_KEY = "hadoop.ssl.enabled";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3e1bff3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index 44ed223..3ffe485 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -29,6 +29,7 @@ import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
@@ -66,6 +67,8 @@ public class AppBlock extends HtmlBlock {
protected ApplicationBaseProtocol appBaseProt;
protected Configuration conf;
protected ApplicationId appID = null;
+ private boolean unsecuredUI = true;
+
@Inject
protected AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx,
@@ -73,6 +76,9 @@ public class AppBlock extends HtmlBlock {
super(ctx);
this.appBaseProt = appBaseProt;
this.conf = conf;
+ // check if UI is unsecured.
+ String httpAuth = conf.get(CommonConfigurationKeys.HADOOP_HTTP_AUTHENTICATION_TYPE);
+ this.unsecuredUI = (httpAuth != null) && httpAuth.equals("simple");
}
@Override
@@ -125,10 +131,16 @@ public class AppBlock extends HtmlBlock {
setTitle(join("Application ", aid));
+ // YARN-6890. for secured cluster allow anonymous UI access, application kill
+ // shouldn't be there.
+ boolean unsecuredUIForSecuredCluster = UserGroupInformation.isSecurityEnabled()
+ && this.unsecuredUI;
+
if (webUiType != null
&& webUiType.equals(YarnWebParams.RM_WEB_UI)
&& conf.getBoolean(YarnConfiguration.RM_WEBAPP_UI_ACTIONS_ENABLED,
- YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED)) {
+ YarnConfiguration.DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED)
+ && !unsecuredUIForSecuredCluster) {
// Application Kill
html.div()
.button()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: YARN-7087. NM failed to perform log
aggregation due to absent container. Contributed by Jason Lowe.
Posted by ju...@apache.org.
YARN-7087. NM failed to perform log aggregation due to absent container. Contributed by Jason Lowe.
(cherry picked from commit e864f81471407a384395fefe1ceb3b66fc7f87f2)
(cherry picked from commit 8eb39f775c09b255986a71eb375c13c0bbc74f6b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/615c661c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/615c661c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/615c661c
Branch: refs/heads/branch-2.8.2
Commit: 615c661c3050a79eacf32e4e185d10f381aae070
Parents: e3e1bff
Author: Eric Payne <ep...@apache.org>
Authored: Fri Aug 25 15:37:54 2017 -0500
Committer: Junping Du <ju...@apache.org>
Committed: Thu Aug 31 14:29:09 2017 -0700
----------------------------------------------------------------------
.../container/ContainerImpl.java | 2 +-
.../logaggregation/LogAggregationService.java | 13 +--
.../event/LogHandlerContainerFinishedEvent.java | 9 +-
.../TestLogAggregationService.java | 105 +++++++++----------
.../TestNonAggregatingLogHandler.java | 10 +-
5 files changed, 69 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/615c661c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 9d1d3c7..7dc8ab6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -495,7 +495,7 @@ public class ContainerImpl implements Container {
eventHandler.handle(new ContainerStopMonitoringEvent(containerId));
// Tell the logService too
eventHandler.handle(new LogHandlerContainerFinishedEvent(
- containerId, exitCode));
+ containerId, containerTokenIdentifier.getContainerType(), exitCode));
}
@SuppressWarnings("unchecked") // dispatcher not typed
http://git-wip-us.apache.org/repos/asf/hadoop/blob/615c661c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
index de89a96..5bf9dbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
@@ -56,7 +56,6 @@ import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
@@ -426,7 +425,8 @@ public class LogAggregationService extends AbstractService implements
return this.appLogAggregators.size();
}
- private void stopContainer(ContainerId containerId, int exitCode) {
+ private void stopContainer(ContainerId containerId,
+ ContainerType containerType, int exitCode) {
// A container is complete. Put this containers' logs up for aggregation if
// this containers' logs are needed.
@@ -437,14 +437,6 @@ public class LogAggregationService extends AbstractService implements
+ ", did it fail to start?");
return;
}
- Container container = context.getContainers().get(containerId);
- if (null == container) {
- LOG.warn("Log aggregation cannot be started for " + containerId
- + ", as its an absent container");
- return;
- }
- ContainerType containerType =
- container.getContainerTokenIdentifier().getContainerType();
aggregator.startContainerLogAggregation(
new ContainerLogContext(containerId, containerType, exitCode));
}
@@ -482,6 +474,7 @@ public class LogAggregationService extends AbstractService implements
LogHandlerContainerFinishedEvent containerFinishEvent =
(LogHandlerContainerFinishedEvent) event;
stopContainer(containerFinishEvent.getContainerId(),
+ containerFinishEvent.getContainerType(),
containerFinishEvent.getExitCode());
break;
case APPLICATION_FINISHED:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/615c661c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerContainerFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerContainerFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerContainerFinishedEvent.java
index 038006e..3f4b6a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerContainerFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/event/LogHandlerContainerFinishedEvent.java
@@ -19,16 +19,19 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.api.ContainerType;
public class LogHandlerContainerFinishedEvent extends LogHandlerEvent {
private final ContainerId containerId;
+ private final ContainerType containerType;
private final int exitCode;
public LogHandlerContainerFinishedEvent(ContainerId containerId,
- int exitCode) {
+ ContainerType containerType, int exitCode) {
super(LogHandlerEventType.CONTAINER_FINISHED);
this.containerId = containerId;
+ this.containerType = containerType;
this.exitCode = exitCode;
}
@@ -36,6 +39,10 @@ public class LogHandlerContainerFinishedEvent extends LogHandlerEvent {
return this.containerId;
}
+ public ContainerType getContainerType() {
+ return containerType;
+ }
+
public int getExitCode() {
return this.exitCode;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/615c661c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 0667d19..4afe57f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -133,7 +133,6 @@ import org.mortbay.util.MultiException;
import com.google.common.base.Supplier;
-//@Ignore
public class TestLogAggregationService extends BaseContainerManagerTest {
private Map<ApplicationAccessType, String> acls = createAppAcls();
@@ -199,13 +198,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
ApplicationAttemptId appAttemptId =
BuilderUtils.newApplicationAttemptId(application1, 1);
- ContainerId container11 = createContainer(appAttemptId, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId container11 = ContainerId.newContainerId(appAttemptId, 1);
// Simulate log-file creation
writeContainerLogs(app1LogDir, container11, new String[] { "stdout",
"stderr", "syslog" });
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container11, 0));
+ new LogHandlerContainerFinishedEvent(container11,
+ ContainerType.APPLICATION_MASTER, 0));
logAggregationService.handle(new LogHandlerAppFinishedEvent(
application1));
@@ -321,11 +320,11 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
ApplicationAttemptId appAttemptId =
BuilderUtils.newApplicationAttemptId(app, 1);
- ContainerId cont = createContainer(appAttemptId, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId cont = ContainerId.newContainerId(appAttemptId, 1);
writeContainerLogs(appLogDir, cont, new String[] { "stdout",
"stderr", "syslog" });
- logAggregationService.handle(new LogHandlerContainerFinishedEvent(cont, 0));
+ logAggregationService.handle(new LogHandlerContainerFinishedEvent(cont,
+ ContainerType.APPLICATION_MASTER, 0));
logAggregationService.handle(new LogHandlerAppFinishedEvent(app));
logAggregationService.stop();
delSrvc.stop();
@@ -407,13 +406,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
ApplicationAttemptId appAttemptId1 =
BuilderUtils.newApplicationAttemptId(application1, 1);
- ContainerId container11 = createContainer(appAttemptId1, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId container11 = ContainerId.newContainerId(appAttemptId1, 1);
// Simulate log-file creation
writeContainerLogs(app1LogDir, container11, fileNames);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container11, 0));
+ new LogHandlerContainerFinishedEvent(container11,
+ ContainerType.APPLICATION_MASTER, 0));
ApplicationId application2 = BuilderUtils.newApplicationId(1234, 2);
ApplicationAttemptId appAttemptId2 =
@@ -430,19 +429,19 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
logAggregationService.handle(new LogHandlerAppStartedEvent(
application2, this.user, null, this.acls, contextWithAMOnly));
- ContainerId container21 = createContainer(appAttemptId2, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId container21 = ContainerId.newContainerId(appAttemptId2, 1);
writeContainerLogs(app2LogDir, container21, fileNames);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container21, 0));
+ new LogHandlerContainerFinishedEvent(container21,
+ ContainerType.APPLICATION_MASTER, 0));
- ContainerId container12 = createContainer(appAttemptId1, 2,
- ContainerType.TASK);
+ ContainerId container12 = ContainerId.newContainerId(appAttemptId1, 2);
writeContainerLogs(app1LogDir, container12, fileNames);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container12, 0));
+ new LogHandlerContainerFinishedEvent(container12,
+ ContainerType.TASK, 0));
ApplicationId application3 = BuilderUtils.newApplicationId(1234, 3);
ApplicationAttemptId appAttemptId3 =
@@ -474,29 +473,29 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
checkEvents(appEventHandler, expectedInitEvents, false, "getType", "getApplicationID");
reset(appEventHandler);
- ContainerId container31 = createContainer(appAttemptId3, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId container31 = ContainerId.newContainerId(appAttemptId3, 1);
writeContainerLogs(app3LogDir, container31, fileNames);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container31, 0));
+ new LogHandlerContainerFinishedEvent(container31,
+ ContainerType.APPLICATION_MASTER, 0));
- ContainerId container32 = createContainer(appAttemptId3, 2,
- ContainerType.TASK);
+ ContainerId container32 = ContainerId.newContainerId(appAttemptId3, 2);
writeContainerLogs(app3LogDir, container32, fileNames);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container32, 1)); // Failed
+ new LogHandlerContainerFinishedEvent(container32,
+ ContainerType.TASK, 1)); // Failed
- ContainerId container22 = createContainer(appAttemptId2, 2,
- ContainerType.TASK);
+ ContainerId container22 = ContainerId.newContainerId(appAttemptId2, 2);
writeContainerLogs(app2LogDir, container22, fileNames);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container22, 0));
+ new LogHandlerContainerFinishedEvent(container22,
+ ContainerType.TASK, 0));
- ContainerId container33 = createContainer(appAttemptId3, 3,
- ContainerType.TASK);
+ ContainerId container33 = ContainerId.newContainerId(appAttemptId3, 3);
writeContainerLogs(app3LogDir, container33, fileNames);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container33, 0));
+ new LogHandlerContainerFinishedEvent(container33,
+ ContainerType.TASK, 0));
logAggregationService.handle(new LogHandlerAppFinishedEvent(
application2));
@@ -750,7 +749,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
// verify trying to collect logs for containers/apps we don't know about
// doesn't blow up and tear down the NM
logAggregationService.handle(new LogHandlerContainerFinishedEvent(
- BuilderUtils.newContainerId(4, 1, 1, 1), 0));
+ BuilderUtils.newContainerId(4, 1, 1, 1),
+ ContainerType.APPLICATION_MASTER, 0));
dispatcher.await();
logAggregationService.handle(new LogHandlerAppFinishedEvent(
BuilderUtils.newApplicationId(1, 5)));
@@ -802,7 +802,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
// verify trying to collect logs for containers/apps we don't know about
// doesn't blow up and tear down the NM
logAggregationService.handle(new LogHandlerContainerFinishedEvent(
- BuilderUtils.newContainerId(4, 1, 1, 1), 0));
+ BuilderUtils.newContainerId(4, 1, 1, 1),
+ ContainerType.APPLICATION_MASTER, 0));
dispatcher.await();
logAggregationService.handle(new LogHandlerAppFinishedEvent(
BuilderUtils.newApplicationId(1, 5)));
@@ -1325,14 +1326,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
ApplicationAttemptId appAttemptId1 =
BuilderUtils.newApplicationAttemptId(application1, 1);
- ContainerId container1 = createContainer(appAttemptId1, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId container1 = ContainerId.newContainerId(appAttemptId1, 1);
// Simulate log-file creation
writeContainerLogs(appLogDir1, container1, new String[] { "stdout",
"stderr", "syslog" });
logAggregationService.handle(new LogHandlerContainerFinishedEvent(
- container1, 0));
+ container1, ContainerType.APPLICATION_MASTER, 0));
// LogContext for application2 has excludePatten which includes
// stdout and syslog.
@@ -1348,13 +1348,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
AMOnlyLogAggregationPolicy.class.getName());
logAggregationService.handle(new LogHandlerAppStartedEvent(application2,
this.user, null, this.acls, LogAggregationContextWithExcludePatterns));
- ContainerId container2 = createContainer(appAttemptId2, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId container2 = ContainerId.newContainerId(appAttemptId2, 1);
writeContainerLogs(app2LogDir, container2, new String[] { "stdout",
"stderr", "syslog" });
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container2, 0));
+ new LogHandlerContainerFinishedEvent(container2,
+ ContainerType.APPLICATION_MASTER, 0));
// LogContext for application3 has includePattern which is *.log and
// excludePatten which includes std.log and sys.log.
@@ -1373,12 +1373,12 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
AMOnlyLogAggregationPolicy.class.getName());
logAggregationService.handle(new LogHandlerAppStartedEvent(application3,
this.user, null, this.acls, context1));
- ContainerId container3 = createContainer(appAttemptId3, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId container3 = ContainerId.newContainerId(appAttemptId3, 1);
writeContainerLogs(app3LogDir, container3, new String[] { "stdout",
"sys.log", "std.log", "out.log", "err.log", "log" });
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container3, 0));
+ new LogHandlerContainerFinishedEvent(container3,
+ ContainerType.APPLICATION_MASTER, 0));
// LogContext for application4 has includePattern
// which includes std.log and sys.log and
@@ -1398,12 +1398,12 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
AMOnlyLogAggregationPolicy.class.getName());
logAggregationService.handle(new LogHandlerAppStartedEvent(application4,
this.user, null, this.acls, context2));
- ContainerId container4 = createContainer(appAttemptId4, 1,
- ContainerType.APPLICATION_MASTER);
+ ContainerId container4 = ContainerId.newContainerId(appAttemptId4, 1);
writeContainerLogs(app4LogDir, container4, new String[] { "stdout",
"sys.log", "std.log", "out.log", "err.log", "log" });
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container4, 0));
+ new LogHandlerContainerFinishedEvent(container4,
+ ContainerType.APPLICATION_MASTER, 0));
dispatcher.await();
ApplicationEvent expectedInitEvents[] =
@@ -1530,7 +1530,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
new ContainerId[] {container}, logFiles, 1, true);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container, 0));
+ new LogHandlerContainerFinishedEvent(container,
+ ContainerType.APPLICATION_MASTER, 0));
dispatcher.await();
@@ -1654,14 +1655,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
ApplicationAttemptId appAttemptId1 =
BuilderUtils.newApplicationAttemptId(appId, 1);
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId1, 2l);
- try {
- logAggregationService.handle(new LogHandlerContainerFinishedEvent(
- containerId, 100));
- assertTrue("Should skip when null containerID", true);
- } catch (Exception e) {
- Assert.assertFalse("Exception not expected should skip null containerid",
- true);
- }
+ logAggregationService.handle(new LogHandlerContainerFinishedEvent(
+ containerId, ContainerType.APPLICATION_MASTER, 100));
}
@Test (timeout = 50000)
@@ -1986,8 +1981,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
long cId, int exitCode, String[] logFiles) throws IOException {
ApplicationAttemptId appAttemptId1 =
BuilderUtils.newApplicationAttemptId(application1, 1);
- ContainerId containerId = createContainer(appAttemptId1, cId,
- containerType);
+ ContainerId containerId = ContainerId.newContainerId(appAttemptId1, cId);
// Simulate log-file creation
File appLogDir1 =
new File(localLogDir, application1.toString());
@@ -1995,7 +1989,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
writeContainerLogs(appLogDir1, containerId, logFiles);
logAggregationService.handle(new LogHandlerContainerFinishedEvent(
- containerId, exitCode));
+ containerId, containerType, exitCode));
return containerId;
}
@@ -2185,7 +2179,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
writeContainerLogs(appLogDir, container, logFiles3);
logAggregationService.handle(
- new LogHandlerContainerFinishedEvent(container, 0));
+ new LogHandlerContainerFinishedEvent(container,
+ ContainerType.APPLICATION_MASTER, 0));
dispatcher.await();
logAggregationService.handle(new LogHandlerAppFinishedEvent(application));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/615c661c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
index ec3757e..591021f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.event.InlineDispatcher;
+import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -152,7 +153,8 @@ public class TestNonAggregatingLogHandler {
logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));
- logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
+ logHandler.handle(new LogHandlerContainerFinishedEvent(container11,
+ ContainerType.APPLICATION_MASTER, 0));
logHandler.handle(new LogHandlerAppFinishedEvent(appId));
@@ -192,7 +194,8 @@ public class TestNonAggregatingLogHandler {
logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));
- logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
+ logHandler.handle(new LogHandlerContainerFinishedEvent(container11,
+ ContainerType.APPLICATION_MASTER, 0));
logHandler.handle(new LogHandlerAppFinishedEvent(appId));
@@ -361,7 +364,8 @@ public class TestNonAggregatingLogHandler {
logHandler.start();
logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));
- logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
+ logHandler.handle(new LogHandlerContainerFinishedEvent(container11,
+ ContainerType.APPLICATION_MASTER, 0));
logHandler.handle(new LogHandlerAppFinishedEvent(appId));
// simulate a restart and verify deletion is rescheduled
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: YARN-7051. Avoid concurrent modification
exception in FifoIntraQueuePreemptionPlugin. Contributed by Eric Payne.
Posted by ju...@apache.org.
YARN-7051. Avoid concurrent modification exception in FifoIntraQueuePreemptionPlugin. Contributed by Eric Payne.
(cherry picked from commit 02599bda04e0ef46f4628b006f2430ad63cac97e)
(cherry picked from commit bc7307788c6a73873a7e058b42ce4b8820b26bd1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8dba358a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8dba358a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8dba358a
Branch: refs/heads/branch-2.8.2
Commit: 8dba358a2904ac7733d0bed6a97baeb6c0bc9299
Parents: 615c661
Author: Sunil G <su...@apache.org>
Authored: Mon Aug 28 12:22:56 2017 +0530
Committer: Junping Du <ju...@apache.org>
Committed: Thu Aug 31 14:29:27 2017 -0700
----------------------------------------------------------------------
.../FifoIntraQueuePreemptionPlugin.java | 26 +++++++++++++-------
1 file changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8dba358a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
index 4bf6760..00ae3da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
@@ -397,10 +397,16 @@ public class FifoIntraQueuePreemptionPlugin
ResourceUsage userResourceUsage = tq.leafQueue.getUser(userName)
.getResourceUsage();
+ // perUserAMUsed was populated with running apps, now we are looping
+ // through both running and pending apps.
+ Resource userSpecificAmUsed = perUserAMUsed.get(userName);
+ amUsed = (userSpecificAmUsed == null)
+ ? Resources.none() : userSpecificAmUsed;
+
TempUserPerPartition tmpUser = new TempUserPerPartition(
tq.leafQueue.getUser(userName), tq.queueName,
Resources.clone(userResourceUsage.getUsed(partition)),
- Resources.clone(perUserAMUsed.get(userName)),
+ Resources.clone(userSpecificAmUsed),
Resources.clone(userResourceUsage.getReserved(partition)),
Resources.none());
@@ -547,15 +553,17 @@ public class FifoIntraQueuePreemptionPlugin
Collection<FiCaSchedulerApp> runningApps = leafQueue.getApplications();
Resource amUsed = Resources.createResource(0, 0);
- for (FiCaSchedulerApp app : runningApps) {
- Resource userAMResource = perUserAMUsed.get(app.getUser());
- if (null == userAMResource) {
- userAMResource = Resources.createResource(0, 0);
- perUserAMUsed.put(app.getUser(), userAMResource);
- }
+ synchronized (leafQueue) {
+ for (FiCaSchedulerApp app : runningApps) {
+ Resource userAMResource = perUserAMUsed.get(app.getUser());
+ if (null == userAMResource) {
+ userAMResource = Resources.createResource(0, 0);
+ perUserAMUsed.put(app.getUser(), userAMResource);
+ }
- Resources.addTo(userAMResource, app.getAMResource(partition));
- Resources.addTo(amUsed, app.getAMResource(partition));
+ Resources.addTo(userAMResource, app.getAMResource(partition));
+ Resources.addTo(amUsed, app.getAMResource(partition));
+ }
}
return amUsed;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org