You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/08/01 17:06:20 UTC
[04/50] hadoop git commit: YARN-8330. Improved publishing ALLOCATED
events to ATS. Contributed by Suma Shivaprasad
YARN-8330. Improved publishing ALLOCATED events to ATS.
Contributed by Suma Shivaprasad
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f93ecf5c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f93ecf5c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f93ecf5c
Branch: refs/heads/HDFS-12943
Commit: f93ecf5c1e0b3db27424963814fc01ec43eb76e0
Parents: e95c5e9
Author: Eric Yang <ey...@apache.org>
Authored: Wed Jul 25 18:49:30 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Jul 25 18:49:30 2018 -0400
----------------------------------------------------------------------
.../rmcontainer/RMContainerImpl.java | 64 +++++++++++---------
.../rmcontainer/TestRMContainerImpl.java | 11 +++-
2 files changed, 43 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f93ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index efac666..945e7cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -244,23 +244,13 @@ public class RMContainerImpl implements RMContainer {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
- saveNonAMContainerMetaInfo = rmContext.getYarnConfiguration().getBoolean(
- YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
- YarnConfiguration
- .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO);
+ saveNonAMContainerMetaInfo =
+ shouldPublishNonAMContainerEventstoATS(rmContext);
if (container.getId() != null) {
rmContext.getRMApplicationHistoryWriter().containerStarted(this);
}
- // If saveNonAMContainerMetaInfo is true, store system metrics for all
- // containers. If false, and if this container is marked as the AM, metrics
- // will still be published for this container, but that calculation happens
- // later.
- if (saveNonAMContainerMetaInfo && null != container.getId()) {
- rmContext.getSystemMetricsPublisher().containerCreated(
- this, this.creationTime);
- }
if (this.container != null) {
this.allocationTags = this.container.getAllocationTags();
}
@@ -590,8 +580,12 @@ public class RMContainerImpl implements RMContainer {
container.getNodeId(), container.getContainerId(),
container.getAllocationTags());
- container.eventHandler.handle(new RMAppAttemptEvent(
- container.appAttemptId, RMAppAttemptEventType.CONTAINER_ALLOCATED));
+ container.eventHandler.handle(
+ new RMAppAttemptEvent(container.appAttemptId,
+ RMAppAttemptEventType.CONTAINER_ALLOCATED));
+
+ publishNonAMContainerEventstoATS(container);
+
}
}
@@ -610,9 +604,11 @@ public class RMContainerImpl implements RMContainer {
// Tell the app
container.eventHandler.handle(new RMAppRunningOnNodeEvent(container
.getApplicationAttemptId().getApplicationId(), container.nodeId));
+
+ publishNonAMContainerEventstoATS(container);
}
}
-
+
private static final class ContainerAcquiredWhileRunningTransition extends
BaseTransition {
@@ -718,17 +714,12 @@ public class RMContainerImpl implements RMContainer {
container);
boolean saveNonAMContainerMetaInfo =
- container.rmContext.getYarnConfiguration().getBoolean(
- YarnConfiguration
- .APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
- YarnConfiguration
- .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO);
+ shouldPublishNonAMContainerEventstoATS(container.rmContext);
if (saveNonAMContainerMetaInfo || container.isAMContainer()) {
container.rmContext.getSystemMetricsPublisher().containerFinished(
container, container.finishTime);
}
-
}
private static void updateAttemptMetrics(RMContainerImpl container) {
@@ -754,6 +745,29 @@ public class RMContainerImpl implements RMContainer {
}
}
+ private static boolean shouldPublishNonAMContainerEventstoATS(
+ RMContext rmContext) {
+ return rmContext.getYarnConfiguration().getBoolean(
+ YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
+ YarnConfiguration
+ .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO);
+ }
+
+ private static void publishNonAMContainerEventstoATS(
+ RMContainerImpl rmContainer) {
+ boolean saveNonAMContainerMetaInfo = shouldPublishNonAMContainerEventstoATS(
+ rmContainer.rmContext);
+
+ // If saveNonAMContainerMetaInfo is true, store system metrics for all
+ // containers. If false, and if this container is marked as the AM, metrics
+ // will still be published for this container, but that calculation happens
+ // later.
+ if (saveNonAMContainerMetaInfo && null != rmContainer.container.getId()) {
+ rmContainer.rmContext.getSystemMetricsPublisher().containerCreated(
+ rmContainer, rmContainer.creationTime);
+ }
+ }
+
private static final class KillTransition extends FinishedTransition {
@Override
@@ -884,13 +898,5 @@ public class RMContainerImpl implements RMContainer {
if (containerId != null) {
rmContext.getRMApplicationHistoryWriter().containerStarted(this);
}
- // If saveNonAMContainerMetaInfo is true, store system metrics for all
- // containers. If false, and if this container is marked as the AM, metrics
- // will still be published for this container, but that calculation happens
- // later.
- if (saveNonAMContainerMetaInfo && null != container.getId()) {
- rmContext.getSystemMetricsPublisher().containerCreated(
- this, this.creationTime);
- }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f93ecf5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 1115e8c..bb6591b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -135,7 +135,6 @@ public class TestRMContainerImpl {
assertEquals(priority,
rmContainer.getAllocatedSchedulerKey().getPriority());
verify(writer).containerStarted(any(RMContainer.class));
- verify(publisher).containerCreated(any(RMContainer.class), anyLong());
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.START));
@@ -150,6 +149,8 @@ public class TestRMContainerImpl {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+ verify(publisher, times(2)).containerCreated(any(RMContainer.class),
+ anyLong());
assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
@@ -240,22 +241,25 @@ public class TestRMContainerImpl {
assertEquals(priority,
rmContainer.getAllocatedSchedulerKey().getPriority());
verify(writer).containerStarted(any(RMContainer.class));
- verify(publisher).containerCreated(any(RMContainer.class), anyLong());
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.START));
drainDispatcher.await();
assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
+ verify(publisher).containerCreated(any(RMContainer.class), anyLong());
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.ACQUIRED));
drainDispatcher.await();
assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());
+ verify(publisher, times(2)).containerCreated(any(RMContainer.class),
+ anyLong());
rmContainer.handle(new RMContainerEvent(containerId,
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+
assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
@@ -340,7 +344,8 @@ public class TestRMContainerImpl {
// RMContainer should be publishing system metrics for all containers.
// Since there is 1 AM container and 1 non-AM container, there should be 2
// container created events and 2 container finished events.
- verify(publisher, times(2)).containerCreated(any(RMContainer.class), anyLong());
+ verify(publisher, times(4)).containerCreated(any(RMContainer.class),
+ anyLong());
verify(publisher, times(2)).containerFinished(any(RMContainer.class), anyLong());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org