You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2012/04/19 01:35:42 UTC
svn commit: r1327724 [4/4] - in
/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-a...
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Wed Apr 18 23:35:30 2012
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.util.HashMap;
import java.util.Map;
@@ -27,7 +28,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -60,9 +60,9 @@ public class TestAMAuthorization {
private static final Log LOG = LogFactory.getLog(TestAMAuthorization.class);
- private static final class MyContainerManager implements ContainerManager {
+ public static final class MyContainerManager implements ContainerManager {
- Map<String, String> containerEnv;
+ public Map<String, String> amContainerEnv;
public MyContainerManager() {
}
@@ -71,7 +71,7 @@ public class TestAMAuthorization {
public StartContainerResponse
startContainer(StartContainerRequest request)
throws YarnRemoteException {
- containerEnv = request.getContainerLaunchContext().getEnvironment();
+ amContainerEnv = request.getContainerLaunchContext().getEnvironment();
return null;
}
@@ -90,19 +90,15 @@ public class TestAMAuthorization {
}
}
- private static class MockRMWithAMS extends MockRMWithCustomAMLauncher {
+ public static class MockRMWithAMS extends MockRMWithCustomAMLauncher {
- private static final Configuration conf = new Configuration();
- static {
+ public MockRMWithAMS(Configuration conf, ContainerManager containerManager) {
+ super(conf, containerManager);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
}
- public MockRMWithAMS(ContainerManager containerManager) {
- super(conf, containerManager);
- }
-
@Override
protected void doSecureLogin() throws IOException {
// Skip the login.
@@ -111,15 +107,14 @@ public class TestAMAuthorization {
@Override
protected ApplicationMasterService createApplicationMasterService() {
- return new ApplicationMasterService(getRMContext(),
- this.appTokenSecretManager, this.scheduler);
+ return new ApplicationMasterService(getRMContext(), this.scheduler);
}
}
@Test
public void testAuthorizedAccess() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- MockRM rm = new MockRMWithAMS(containerManager);
+ final MockRM rm = new MockRMWithAMS(new Configuration(), containerManager);
rm.start();
MockNM nm1 = rm.registerNode("localhost:1234", 5120);
@@ -132,11 +127,11 @@ public class TestAMAuthorization {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.containerEnv == null && waitCount++ < 20) {
+ while (containerManager.amContainerEnv == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.containerEnv);
+ Assert.assertNotNull(containerManager.amContainerEnv);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -145,13 +140,10 @@ public class TestAMAuthorization {
// Create a client to the RM.
final Configuration conf = rm.getConfig();
final YarnRPC rpc = YarnRPC.create(conf);
- final String serviceAddr = conf.get(
- YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);
UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- String tokenURLEncodedStr = containerManager.containerEnv
+ String tokenURLEncodedStr = containerManager.amContainerEnv
.get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
LOG.info("AppMasterToken is " + tokenURLEncodedStr);
Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
@@ -162,8 +154,8 @@ public class TestAMAuthorization {
.doAs(new PrivilegedAction<AMRMProtocol>() {
@Override
public AMRMProtocol run() {
- return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, NetUtils
- .createSocketAddr(serviceAddr), conf);
+ return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rm
+ .getApplicationMasterService().getBindAddress(), conf);
}
});
@@ -181,7 +173,7 @@ public class TestAMAuthorization {
@Test
public void testUnauthorizedAccess() throws Exception {
MyContainerManager containerManager = new MyContainerManager();
- MockRM rm = new MockRMWithAMS(containerManager);
+ MockRM rm = new MockRMWithAMS(new Configuration(), containerManager);
rm.start();
MockNM nm1 = rm.registerNode("localhost:1234", 5120);
@@ -191,11 +183,11 @@ public class TestAMAuthorization {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.containerEnv == null && waitCount++ < 20) {
+ while (containerManager.amContainerEnv == null && waitCount++ < 20) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
- Assert.assertNotNull(containerManager.containerEnv);
+ Assert.assertNotNull(containerManager.amContainerEnv);
RMAppAttempt attempt = app.getCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
@@ -204,13 +196,14 @@ public class TestAMAuthorization {
// Create a client to the RM.
final Configuration conf = rm.getConfig();
final YarnRPC rpc = YarnRPC.create(conf);
- final String serviceAddr = conf.get(
+ final InetSocketAddress serviceAddr = conf.getSocketAddr(
YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(applicationAttemptId.toString());
- String tokenURLEncodedStr = containerManager.containerEnv
+ String tokenURLEncodedStr = containerManager.amContainerEnv
.get(ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
LOG.info("AppMasterToken is " + tokenURLEncodedStr);
Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
@@ -221,8 +214,8 @@ public class TestAMAuthorization {
.doAs(new PrivilegedAction<AMRMProtocol>() {
@Override
public AMRMProtocol run() {
- return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, NetUtils
- .createSocketAddr(serviceAddr), conf);
+ return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class,
+ serviceAddr, conf);
}
});
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java Wed Apr 18 23:35:30 2012
@@ -27,8 +27,8 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.MockApps;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
@@ -93,7 +92,7 @@ public class TestAppManager{
AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(
rmDispatcher);
return new RMContextImpl(new MemStore(), rmDispatcher,
- containerAllocationExpirer, amLivelinessMonitor, null) {
+ containerAllocationExpirer, amLivelinessMonitor, null, null) {
@Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return map;
@@ -336,9 +335,9 @@ public class TestAppManager{
RMContext rmContext = mockRMContext(0, now - 10);
ResourceScheduler scheduler = new CapacityScheduler();
- ApplicationMasterService masterService = new ApplicationMasterService(rmContext,
- new ApplicationTokenSecretManager(), scheduler);
Configuration conf = new Configuration();
+ ApplicationMasterService masterService =
+ new ApplicationMasterService(rmContext, scheduler);
TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
new ClientToAMSecretManager(), scheduler, masterService,
new ApplicationACLsManager(conf), conf);
@@ -384,9 +383,9 @@ public class TestAppManager{
RMContext rmContext = mockRMContext(1, now - 10);
ResourceScheduler scheduler = new CapacityScheduler();
- ApplicationMasterService masterService = new ApplicationMasterService(rmContext,
- new ApplicationTokenSecretManager(), scheduler);
Configuration conf = new Configuration();
+ ApplicationMasterService masterService =
+ new ApplicationMasterService(rmContext, scheduler);
TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
new ClientToAMSecretManager(), scheduler, masterService,
new ApplicationACLsManager(conf), conf);
@@ -432,9 +431,9 @@ public class TestAppManager{
// specify 1 here and use same appId below so it gets duplicate entry
RMContext rmContext = mockRMContext(1, now - 10);
ResourceScheduler scheduler = new CapacityScheduler();
- ApplicationMasterService masterService = new ApplicationMasterService(rmContext,
- new ApplicationTokenSecretManager(), scheduler);
Configuration conf = new Configuration();
+ ApplicationMasterService masterService =
+ new ApplicationMasterService(rmContext, scheduler);
TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
new ClientToAMSecretManager(), scheduler, masterService,
new ApplicationACLsManager(conf), conf);
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Wed Apr 18 23:35:30 2012
@@ -30,7 +30,6 @@ import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.ClientRMProtocol;
@@ -75,9 +74,10 @@ public class TestApplicationACLs {
static MockRM resourceManager;
static Configuration conf = new YarnConfiguration();
final static YarnRPC rpc = YarnRPC.create(conf);
- final static InetSocketAddress rmAddress = NetUtils
- .createSocketAddr(conf.get(YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS));
+ final static InetSocketAddress rmAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
private static ClientRMProtocol rmClient;
private static RecordFactory recordFactory = RecordFactoryProvider
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Wed Apr 18 23:35:30 2012
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import java.io.IOException;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -33,7 +32,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher;
@@ -123,14 +121,13 @@ public class TestApplicationMasterLaunch
@Override
protected ApplicationMasterLauncher createAMLauncher() {
- return new ApplicationMasterLauncher(super.appTokenSecretManager,
- super.clientToAMSecretManager, getRMContext()) {
+ return new ApplicationMasterLauncher(super.clientToAMSecretManager,
+ getRMContext()) {
@Override
protected Runnable createRunnableLauncher(RMAppAttempt application,
AMLauncherEventType event) {
return new AMLauncher(context, application, event,
- applicationTokenSecretManager, clientToAMSecretManager,
- getConfig()) {
+ clientToAMSecretManager, getConfig()) {
@Override
protected ContainerManager getContainerMgrProxy(
ContainerId containerId) {
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java Wed Apr 18 23:35:30 2012
@@ -78,7 +78,7 @@ public class TestRMNodeTransitions {
rmContext =
new RMContextImpl(new MemStore(), rmDispatcher, null, null,
- mock(DelegationTokenRenewer.class));
+ mock(DelegationTokenRenewer.class), null);
scheduler = mock(YarnScheduler.class);
doAnswer(
new Answer<Void>() {
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java Wed Apr 18 23:35:30 2012
@@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
@@ -55,6 +54,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Wed Apr 18 23:35:30 2012
@@ -48,7 +48,6 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
@@ -62,6 +61,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.junit.After;
import org.junit.Before;
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java Wed Apr 18 23:35:30 2012
@@ -73,7 +73,7 @@ public class TestNMExpiry {
// Dispatcher that processes events inline
Dispatcher dispatcher = new InlineDispatcher();
RMContext context = new RMContextImpl(new MemStore(), dispatcher, null,
- null, null);
+ null, null, null);
dispatcher.register(SchedulerEventType.class,
new InlineDispatcher.EmptyEventHandler());
dispatcher.register(RMNodeEventType.class,
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java Wed Apr 18 23:35:30 2012
@@ -67,7 +67,7 @@ public class TestRMNMRPCResponseId {
}
});
RMContext context =
- new RMContextImpl(new MemStore(), dispatcher, null, null, null);
+ new RMContextImpl(new MemStore(), dispatcher, null, null, null, null);
dispatcher.register(RMNodeEventType.class,
new ResourceManager.NodeEventDispatcher(context));
NodesListManager nodesListManager = new NodesListManager(context);
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Wed Apr 18 23:35:30 2012
@@ -34,7 +34,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
@@ -48,7 +47,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
-
+import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.junit.Before;
import org.junit.Test;
@@ -118,8 +117,10 @@ public class TestRMAppTransitions {
ContainerAllocationExpirer containerAllocationExpirer =
mock(ContainerAllocationExpirer.class);
AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
- this.rmContext = new RMContextImpl(new MemStore(), rmDispatcher,
- containerAllocationExpirer, amLivelinessMonitor, null);
+ this.rmContext =
+ new RMContextImpl(new MemStore(), rmDispatcher,
+ containerAllocationExpirer, amLivelinessMonitor, null,
+ new ApplicationTokenSecretManager(conf));
rmDispatcher.register(RMAppAttemptEventType.class,
new TestApplicationAttemptEventDispatcher(this.rmContext));
@@ -142,9 +143,8 @@ public class TestRMAppTransitions {
String clientTokenStr = "bogusstring";
ApplicationStore appStore = mock(ApplicationStore.class);
YarnScheduler scheduler = mock(YarnScheduler.class);
- ApplicationMasterService masterService =
- new ApplicationMasterService(rmContext,
- new ApplicationTokenSecretManager(), scheduler);
+ ApplicationMasterService masterService =
+ new ApplicationMasterService(rmContext, scheduler);
RMApp application = new RMAppImpl(applicationId, rmContext,
conf, name, user,
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Wed Apr 18 23:35:30 2012
@@ -17,9 +17,15 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.*;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.util.Collections;
import java.util.List;
@@ -61,6 +67,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -136,8 +143,10 @@ public class TestRMAppAttemptTransitions
ContainerAllocationExpirer containerAllocationExpirer =
mock(ContainerAllocationExpirer.class);
AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
- rmContext = new RMContextImpl(new MemStore(), rmDispatcher,
- containerAllocationExpirer, amLivelinessMonitor, null);
+ rmContext =
+ new RMContextImpl(new MemStore(), rmDispatcher,
+ containerAllocationExpirer, amLivelinessMonitor, null,
+ new ApplicationTokenSecretManager(new Configuration()));
scheduler = mock(YarnScheduler.class);
masterService = mock(ApplicationMasterService.class);
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Wed Apr 18 23:35:30 2012
@@ -927,6 +927,103 @@ public class TestLeafQueue {
}
@Test
+ public void testStolenReservedContainer() throws Exception {
+ // Manipulate queue 'a'
+ LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ //unset maxCapacity
+ a.setMaxCapacity(1.0f);
+
+ // Users
+ final String user_0 = "user_0";
+ final String user_1 = "user_1";
+
+ // Submit applications
+ final ApplicationAttemptId appAttemptId_0 =
+ TestUtils.getMockApplicationAttemptId(0, 0);
+ SchedulerApp app_0 =
+ new SchedulerApp(appAttemptId_0, user_0, a,
+ mock(ActiveUsersManager.class), rmContext, null);
+ a.submitApplication(app_0, user_0, A);
+
+ final ApplicationAttemptId appAttemptId_1 =
+ TestUtils.getMockApplicationAttemptId(1, 0);
+ SchedulerApp app_1 =
+ new SchedulerApp(appAttemptId_1, user_1, a,
+ mock(ActiveUsersManager.class), rmContext, null);
+ a.submitApplication(app_1, user_1, A);
+
+ // Setup some nodes
+ String host_0 = "host_0";
+ SchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 4*GB);
+ String host_1 = "host_1";
+ SchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, 4*GB);
+
+ final int numNodes = 3;
+ Resource clusterResource = Resources.createResource(numNodes * (4*GB));
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+
+ // Setup resource-requests
+ Priority priority = TestUtils.createMockPriority(1);
+ app_0.updateResourceRequests(Collections.singletonList(
+ TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority,
+ recordFactory)));
+
+ // Setup app_1 to request a 4GB container on host_0 and
+ // another 4GB container anywhere.
+ ArrayList<ResourceRequest> appRequests_1 =
+ new ArrayList<ResourceRequest>(4);
+ appRequests_1.add(TestUtils.createResourceRequest(host_0, 4*GB, 1,
+ priority, recordFactory));
+ appRequests_1.add(TestUtils.createResourceRequest(DEFAULT_RACK, 4*GB, 1,
+ priority, recordFactory));
+ appRequests_1.add(TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 2,
+ priority, recordFactory));
+ app_1.updateResourceRequests(appRequests_1);
+
+ // Start testing...
+
+ a.assignContainers(clusterResource, node_0);
+ assertEquals(2*GB, a.getUsedResources().getMemory());
+ assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
+ assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(2*GB, a.getMetrics().getAllocatedMB());
+ assertEquals(0*GB, a.getMetrics().getAvailableMB());
+
+ // Now, reservation should kick in for app_1
+ a.assignContainers(clusterResource, node_0);
+ assertEquals(6*GB, a.getUsedResources().getMemory());
+ assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
+ assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
+ assertEquals(2*GB, node_0.getUsedResource().getMemory());
+ assertEquals(4*GB, a.getMetrics().getReservedMB());
+ assertEquals(2*GB, a.getMetrics().getAllocatedMB());
+
+ // node_1 heartbeats in and gets the DEFAULT_RACK request for app_1
+ a.assignContainers(clusterResource, node_1);
+ assertEquals(10*GB, a.getUsedResources().getMemory());
+ assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
+ assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
+ assertEquals(4*GB, node_1.getUsedResource().getMemory());
+ assertEquals(4*GB, a.getMetrics().getReservedMB());
+ assertEquals(6*GB, a.getMetrics().getAllocatedMB());
+
+ // Now free 1 container from app_0 and try to assign to node_0
+ a.completedContainer(clusterResource, app_0, node_0,
+ app_0.getLiveContainers().iterator().next(), null, RMContainerEventType.KILL);
+ a.assignContainers(clusterResource, node_0);
+ assertEquals(8*GB, a.getUsedResources().getMemory());
+ assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
+ assertEquals(8*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
+ assertEquals(4*GB, node_0.getUsedResource().getMemory());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(8*GB, a.getMetrics().getAllocatedMB());
+ }
+
+ @Test
public void testReservationExchange() throws Exception {
// Manipulate queue 'a'
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Wed Apr 18 23:35:30 2012
@@ -18,10 +18,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import static org.mockito.Mockito.*;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -42,6 +46,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
public class TestUtils {
private static final Log LOG = LogFactory.getLog(TestUtils.class);
@@ -74,8 +79,9 @@ public class TestUtils {
ContainerAllocationExpirer cae =
new ContainerAllocationExpirer(nullDispatcher);
- RMContext rmContext =
- new RMContextImpl(null, nullDispatcher, cae, null, null);
+ RMContext rmContext =
+ new RMContextImpl(null, nullDispatcher, cae, null, null,
+ new ApplicationTokenSecretManager(new Configuration()));
return rmContext;
}
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Wed Apr 18 23:35:30 2012
@@ -85,7 +85,8 @@ public class TestFifoScheduler {
@Test
public void testAppAttemptMetrics() throws Exception {
AsyncDispatcher dispatcher = new InlineDispatcher();
- RMContext rmContext = new RMContextImpl(null, dispatcher, null, null, null);
+ RMContext rmContext =
+ new RMContextImpl(null, dispatcher, null, null, null, null);
FifoScheduler schedular = new FifoScheduler();
schedular.reinitialize(new Configuration(), null, rmContext);
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Wed Apr 18 23:35:30 2012
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
@@ -93,6 +94,11 @@ public class TestRMWebApp {
rmViewInstance.set(YarnWebParams.APP_STATE, RMAppState.RUNNING.toString());
rmViewInstance.render();
WebAppTests.flushOutput(injector);
+
+ rmViewInstance.set(YarnWebParams.APP_STATE, StringHelper.cjoin(
+ RMAppState.ACCEPTED.toString(), RMAppState.RUNNING.toString()));
+ rmViewInstance.render();
+ WebAppTests.flushOutput(injector);
}
@Test public void testNodesPage() {
@@ -152,7 +158,7 @@ public class TestRMWebApp {
for (RMNode node : deactivatedNodes) {
deactivatedNodesMap.put(node.getHostName(), node);
}
- return new RMContextImpl(new MemStore(), null, null, null, null) {
+ return new RMContextImpl(new MemStore(), null, null, null, null, null) {
@Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return applicationsMaps;
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java Wed Apr 18 23:35:30 2012
@@ -55,8 +55,6 @@ import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
-import clover.org.jfree.util.Log;
-
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.servlet.GuiceServletContextListener;
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Wed Apr 18 23:35:30 2012
@@ -78,12 +78,12 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
-import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
@@ -387,20 +387,19 @@ public class TestContainerManagerSecurit
appAttempt.getAppAttemptId().toString());
// Ask for a container from the RM
- String schedulerAddressString = conf.get(
- YarnConfiguration.RM_SCHEDULER_ADDRESS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);
- final InetSocketAddress schedulerAddr = NetUtils
- .createSocketAddr(schedulerAddressString);
+ final InetSocketAddress schedulerAddr =
+ resourceManager.getApplicationMasterService().getBindAddress();
ApplicationTokenIdentifier appTokenIdentifier = new ApplicationTokenIdentifier(
appAttempt.getAppAttemptId());
- ApplicationTokenSecretManager appTokenSecretManager = new ApplicationTokenSecretManager();
- appTokenSecretManager.setMasterKey(ApplicationTokenSecretManager
- .createSecretKey("Dummy".getBytes())); // TODO: FIX. Be in Sync with
- // ResourceManager.java
- Token<ApplicationTokenIdentifier> appToken = new Token<ApplicationTokenIdentifier>(
- appTokenIdentifier, appTokenSecretManager);
- appToken.setService(new Text(schedulerAddressString));
+ ApplicationTokenSecretManager appTokenSecretManager =
+ new ApplicationTokenSecretManager(conf);
+ appTokenSecretManager.setMasterKey(resourceManager
+ .getApplicationTokenSecretManager().getMasterKey());
+ Token<ApplicationTokenIdentifier> appToken =
+ new Token<ApplicationTokenIdentifier>(appTokenIdentifier,
+ appTokenSecretManager);
+ appToken.setService(new Text(schedulerAddr.getHostName() + ":"
+ + schedulerAddr.getPort()));
currentUser.addToken(appToken);
AMRMProtocol scheduler = currentUser
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java Wed Apr 18 23:35:30 2012
@@ -23,7 +23,6 @@ import java.net.InetSocketAddress;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.ClientRMProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
@@ -51,10 +50,10 @@ public class AppReportFetcher {
public AppReportFetcher(Configuration conf) {
this.conf = conf;
YarnRPC rpc = YarnRPC.create(this.conf);
- InetSocketAddress rmAddress =
- NetUtils.createSocketAddr(this.conf.get(
+ InetSocketAddress rmAddress = conf.getSocketAddr(
YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS));
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
LOG.info("Connecting to ResourceManager at " + rmAddress);
applicationsManager =
(ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class,
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/pom.xml?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/pom.xml (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/pom.xml Wed Apr 18 23:35:30 2012
@@ -109,7 +109,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <scope>provided</scope>
+ <scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1324567-1327718
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java Wed Apr 18 23:35:30 2012
@@ -235,7 +235,9 @@ implements ResourceUsageEmulatorPlugin {
@Override
public float getProgress() {
- return Math.min(1f, ((float)getCurrentCPUUsage())/targetCpuUsage);
+ return enabled
+ ? Math.min(1f, ((float)getCurrentCPUUsage())/targetCpuUsage)
+ : 1.0f;
}
@Override
@@ -297,6 +299,9 @@ implements ResourceUsageEmulatorPlugin {
public void initialize(Configuration conf, ResourceUsageMetrics metrics,
ResourceCalculatorPlugin monitor,
Progressive progress) {
+ this.monitor = monitor;
+ this.progress = progress;
+
// get the target CPU usage
targetCpuUsage = metrics.getCumulativeCpuUsage();
if (targetCpuUsage <= 0 ) {
@@ -306,8 +311,6 @@ implements ResourceUsageEmulatorPlugin {
enabled = true;
}
- this.monitor = monitor;
- this.progress = progress;
emulationInterval = conf.getFloat(CPU_EMULATION_PROGRESS_INTERVAL,
DEFAULT_EMULATION_FREQUENCY);
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java Wed Apr 18 23:35:30 2012
@@ -188,7 +188,9 @@ implements ResourceUsageEmulatorPlugin {
@Override
public float getProgress() {
- return Math.min(1f, ((float)getTotalHeapUsageInMB())/targetHeapUsageInMB);
+ return enabled
+ ? Math.min(1f, ((float)getTotalHeapUsageInMB())/targetHeapUsageInMB)
+ : 1.0f;
}
@Override
@@ -237,6 +239,8 @@ implements ResourceUsageEmulatorPlugin {
public void initialize(Configuration conf, ResourceUsageMetrics metrics,
ResourceCalculatorPlugin monitor,
Progressive progress) {
+ this.progress = progress;
+
// get the target heap usage
targetHeapUsageInMB = metrics.getHeapUsage() / ONE_MB;
if (targetHeapUsageInMB <= 0 ) {
@@ -248,7 +252,6 @@ implements ResourceUsageEmulatorPlugin {
enabled = true;
}
- this.progress = progress;
emulationInterval =
conf.getFloat(HEAP_EMULATION_PROGRESS_INTERVAL,
DEFAULT_EMULATION_PROGRESS_INTERVAL);
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java Wed Apr 18 23:35:30 2012
@@ -171,6 +171,11 @@ public class TestGridmixMemoryEmulation
assertEquals("Disabled heap usage emulation plugin works!",
heapUsagePre, heapUsagePost);
+ // test with get progress
+ float progress = heapPlugin.getProgress();
+ assertEquals("Invalid progress of disabled cumulative heap usage emulation "
+ + "plugin!", 1.0f, progress, 0f);
+
// test with wrong/invalid configuration
Boolean failed = null;
invalidUsage =
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java Wed Apr 18 23:35:30 2012
@@ -32,7 +32,6 @@ import org.apache.hadoop.mapreduce.TaskT
import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
import org.apache.hadoop.mapreduce.task.MapContextImpl;
import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
-import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin.ProcResourceValues;
import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin;
import org.apache.hadoop.mapred.gridmix.LoadJob.ResourceUsageMatcherRunner;
@@ -484,6 +483,11 @@ public class TestResourceUsageEmulators
assertEquals("Disabled cumulative CPU usage emulation plugin works!",
cpuUsagePre, cpuUsagePost);
+ // test with get progress
+ float progress = cpuPlugin.getProgress();
+ assertEquals("Invalid progress of disabled cumulative CPU usage emulation "
+ + "plugin!", 1.0f, progress, 0f);
+
// test with valid resource usage value
ResourceUsageMetrics metrics = createMetrics(targetCpuUsage);
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1324567-1327718
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/mapred-default.xml?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/mapred-default.xml (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/mapred-default.xml Wed Apr 18 23:35:30 2012
@@ -403,9 +403,6 @@
For example, to enable verbose gc logging to a file named for the taskid in
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
-
- The configuration variable mapred.child.ulimit can be used to control the
- maximum virtual memory of the child processes.
</description>
</property>
@@ -420,20 +417,6 @@
</property>
<property>
- <name>mapred.child.ulimit</name>
- <value></value>
- <description>The maximum virtual memory, in KB, of a process launched by the
- Map-Reduce framework. This can be used to control both the Mapper/Reducer
- tasks and applications using Hadoop Pipes, Hadoop Streaming etc.
- By default it is left unspecified to let cluster admins control it via
- limits.conf and other such relevant mechanisms.
-
- Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to
- JavaVM, else the VM might not start.
- </description>
-</property>
-
-<property>
<name>mapreduce.map.log.level</name>
<value>INFO</value>
<description>The logging level for the map task. The allowed levels are:
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/JobInProgress.java Wed Apr 18 23:35:30 2012
@@ -2731,7 +2731,7 @@ public class JobInProgress {
}
TaskFinishedEvent tfe = new TaskFinishedEvent(tip.getTIPId(),
- tip.getExecFinishTime(), taskType,
+ statusAttemptID, tip.getExecFinishTime(), taskType,
TaskStatus.State.SUCCEEDED.toString(),
new org.apache.hadoop.mapreduce.Counters(status.getCounters()));
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java Wed Apr 18 23:35:30 2012
@@ -33,12 +33,6 @@ class MapTaskRunner extends TaskRunner {
super.getChildJavaOpts(jobConf,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
}
-
- @Override
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_MAP_TASK_ULIMIT,
- super.getChildUlimit(jobConf));
- }
@Override
public String getChildEnv(JobConf jobConf) {
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java Wed Apr 18 23:35:30 2012
@@ -41,12 +41,6 @@ class ReduceTaskRunner extends TaskRunne
super.getChildJavaOpts(jobConf,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
}
-
- @Override
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_REDUCE_TASK_ULIMIT,
- super.getChildUlimit(jobConf));
- }
@Override
public String getChildEnv(JobConf jobConf) {
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java Wed Apr 18 23:35:30 2012
@@ -110,20 +110,7 @@ abstract class TaskRunner extends Thread
public String getChildJavaOpts(JobConf jobConf, String defaultValue) {
return jobConf.get(JobConf.MAPRED_TASK_JAVA_OPTS, defaultValue);
}
-
- /**
- * Get the maximum virtual memory of the child map/reduce tasks.
- * @param jobConf job configuration
- * @return the maximum virtual memory of the child task or <code>-1</code> if
- * none is specified
- * @deprecated Use limits specific to the map or reduce tasks set via
- * {@link JobConf#MAPRED_MAP_TASK_ULIMIT} or
- * {@link JobConf#MAPRED_REDUCE_TASK_ULIMIT}
- */
- @Deprecated
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_TASK_ULIMIT, -1);
- }
+
/**
* Get the environment variables for the child map/reduce tasks.
@@ -188,8 +175,7 @@ abstract class TaskRunner extends Thread
tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf);
- // set memory limit using ulimit if feasible and necessary ...
- List<String> setup = getVMSetupCmd();
+ List<String> setup = new ArrayList<String>();
// Set up the redirection of the task's stdout and stderr streams
File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask());
@@ -311,26 +297,6 @@ abstract class TaskRunner extends Thread
}
/**
- * @return
- */
- private List<String> getVMSetupCmd() {
-
- int ulimit = getChildUlimit(conf);
- if (ulimit <= 0) {
- return null;
- }
- List<String> setup = null;
- String[] ulimitCmd = Shell.getUlimitMemoryCommand(ulimit);
- if (ulimitCmd != null) {
- setup = new ArrayList<String>();
- for (String arg : ulimitCmd) {
- setup.add(arg);
- }
- }
- return setup;
- }
-
- /**
* Parse the given string and return an array of individual java opts. Split
* on whitespace and replace the special string "@taskid@" with the task ID
* given.
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1324567-1327718
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1324567-1327718
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistoryParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistoryParsing.java?rev=1327724&r1=1327723&r2=1327724&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistoryParsing.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/TestJobHistoryParsing.java Wed Apr 18 23:35:30 2012
@@ -97,7 +97,7 @@ public class TestJobHistoryParsing exte
// Try to write one more event now, should not fail
TaskID tid = TaskID.forName("task_200809171136_0001_m_000002");
TaskFinishedEvent tfe =
- new TaskFinishedEvent(tid, 0, TaskType.MAP, "", null);
+ new TaskFinishedEvent(tid, null, 0, TaskType.MAP, "", null);
boolean caughtException = false;
try {
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1324567-1327718