You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2015/06/11 06:41:36 UTC
hadoop git commit: YARN-3785. Support for Resource as an argument
during submitApp call in MockRM test class. Contributed by Sunil G
Repository: hadoop
Updated Branches:
refs/heads/trunk a7a776834 -> 5583f88bf
YARN-3785. Support for Resource as an argument during submitApp call in
MockRM test class. Contributed by Sunil G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5583f88b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5583f88b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5583f88b
Branch: refs/heads/trunk
Commit: 5583f88bf7f1852dc0907ce55d0755e4fb22107a
Parents: a7a7768
Author: Xuan <xg...@apache.org>
Authored: Wed Jun 10 21:40:48 2015 -0700
Committer: Xuan <xg...@apache.org>
Committed: Wed Jun 10 21:40:48 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/MockRM.java | 28 +++++++++++++++-----
.../TestWorkPreservingRMRestart.java | 10 ++++---
.../capacity/TestCapacityScheduler.java | 11 +++++---
.../security/TestDelegationTokenRenewer.java | 27 ++++++++++---------
5 files changed, 52 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5583f88b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cd00cf4..dc43ad8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -303,6 +303,9 @@ Release 2.8.0 - UNRELEASED
YARN-3787. Allowed generic history service to load a number of applications whose
started time is within the given range. (Xuan Gong via zjshen)
+ YARN-3785. Support for Resource as an argument during submitApp call in MockRM
+ test class. (Sunil G via xgong)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5583f88b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 672ce13..d068a94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -322,6 +322,14 @@ public class MockRM extends ResourceManager {
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null);
}
+ public RMApp submitApp(Resource resource, String name, String user,
+ Map<ApplicationAccessType, String> acls, String queue) throws Exception {
+ return submitApp(resource, name, user, acls, false, queue,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null,
+ true, false, false, null, 0, null, true);
+ }
+
public RMApp submitApp(int masterMemory, String name, String user,
Map<ApplicationAccessType, String> acls, String queue,
boolean waitForAccepted) throws Exception {
@@ -358,14 +366,18 @@ public class MockRM extends ResourceManager {
Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
int maxAppAttempts, Credentials ts, String appType,
boolean waitForAccepted, boolean keepContainers) throws Exception {
- return submitApp(masterMemory, name, user, acls, unmanaged, queue,
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(masterMemory);
+ return submitApp(resource, name, user, acls, unmanaged, queue,
maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
false, null, 0, null, true);
}
public RMApp submitApp(int masterMemory, long attemptFailuresValidityInterval)
throws Exception {
- return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(masterMemory);
+ return submitApp(resource, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null,
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
@@ -377,21 +389,25 @@ public class MockRM extends ResourceManager {
int maxAppAttempts, Credentials ts, String appType,
boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
ApplicationId applicationId) throws Exception {
- return submitApp(masterMemory, name, user, acls, unmanaged, queue,
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(masterMemory);
+ return submitApp(resource, name, user, acls, unmanaged, queue,
maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
isAppIdProvided, applicationId, 0, null, true);
}
public RMApp submitApp(int masterMemory,
LogAggregationContext logAggregationContext) throws Exception {
- return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(masterMemory);
+ return submitApp(resource, "", UserGroupInformation.getCurrentUser()
.getShortUserName(), null, false, null,
super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false,
false, null, 0, logAggregationContext, true);
}
- public RMApp submitApp(int masterMemory, String name, String user,
+ public RMApp submitApp(Resource capability, String name, String user,
Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
int maxAppAttempts, Credentials ts, String appType,
boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
@@ -422,8 +438,6 @@ public class MockRM extends ResourceManager {
sub.setApplicationType(appType);
ContainerLaunchContext clc = Records
.newRecord(ContainerLaunchContext.class);
- final Resource capability = Records.newRecord(Resource.class);
- capability.setMemory(masterMemory);
sub.setResource(capability);
clc.setApplicationACLs(acls);
if (ts != null && UserGroupInformation.isSecurityEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5583f88b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index c6fe371..32743c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -78,6 +78,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.Dom
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -1050,9 +1051,12 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
nm1.registerNode();
// submit app with keepContainersAcrossApplicationAttempts true
- RMApp app0 = rm1.submitApp(200, "", UserGroupInformation.getCurrentUser()
- .getShortUserName(), null, false, null, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS,
- null, null, true, true, false, null, 0, null, true);
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(200);
+ RMApp app0 = rm1.submitApp(resource, "", UserGroupInformation
+ .getCurrentUser().getShortUserName(), null, false, null,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true,
+ false, null, 0, null, true);
MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);
am0.allocate("127.0.0.1", 1000, 2, new ArrayList<ContainerId>());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5583f88b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 3827f85..e8afab2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -2986,13 +2986,16 @@ public class TestCapacityScheduler {
(LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
Resource amResourceLimit = queueA.getAMResourceLimit();
- Resource amResource =
- Resource.newInstance(amResourceLimit.getMemory() + 1,
+ Resource amResource1 =
+ Resource.newInstance(amResourceLimit.getMemory() + 1024,
+ amResourceLimit.getVirtualCores() + 1);
+ Resource amResource2 =
+ Resource.newInstance(amResourceLimit.getMemory() + 2048,
amResourceLimit.getVirtualCores() + 1);
- rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+ rm.submitApp(amResource1, "app-1", userName, null, queueName);
- rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+ rm.submitApp(amResource2, "app-2", userName, null, queueName);
// When AM limit is exceeded, 1 applications will be activated.Rest all
// applications will be in pending
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5583f88b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index 10e840b..49c7bf9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer.DelegationTokenToRenew;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -1044,16 +1045,16 @@ public class TestDelegationTokenRenewer {
credentials.addToken(userText1, token1);
// submit app1 with a token, set cancelTokenWhenComplete to false;
- RMApp app1 =
- rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
- null, true, false, false, null, 0, null, false);
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(200);
+ RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2,
+ credentials, null, true, false, false, null, 0, null, false);
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
// submit app2 with the same token, set cancelTokenWhenComplete to true;
- RMApp app2 =
- rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
- null, true, false, false, null, 0, null, true);
+ RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
+ credentials, null, true, false, false, null, 0, null, true);
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
@@ -1109,8 +1110,10 @@ public class TestDelegationTokenRenewer {
Assert.assertTrue(renewer.getAllTokens().isEmpty());
Assert.assertFalse(Renewer.cancelled);
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(200);
RMApp app1 =
- rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
+ rm.submitApp(resource, "name", "user", null, false, null, 2, credentials,
null, true, false, false, null, 0, null, true);
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);
@@ -1118,9 +1121,8 @@ public class TestDelegationTokenRenewer {
DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1);
Assert.assertNotNull(dttr);
Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
- RMApp app2 =
- rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
- null, true, false, false, null, 0, null, true);
+ RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
+ credentials, null, true, false, false, null, 0, null, true);
MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
@@ -1136,9 +1138,8 @@ public class TestDelegationTokenRenewer {
Assert.assertFalse(dttr.isTimerCancelled());
Assert.assertFalse(Renewer.cancelled);
- RMApp app3 =
- rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
- null, true, false, false, null, 0, null, true);
+ RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2,
+ credentials, null, true, false, false, null, 0, null, true);
MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1);
rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING);
Assert.assertTrue(renewer.getAllTokens().containsKey(token1));