You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by to...@apache.org on 2013/01/23 19:51:28 UTC
svn commit: r1437623 [3/3] - in
/hadoop/common/branches/HDFS-347/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/...
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java Wed Jan 23 18:51:24 2013
@@ -18,10 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationStateDataPBImpl;
+@Unstable
public class NullRMStateStore extends RMStateStore {
@Override
@@ -36,7 +38,7 @@ public class NullRMStateStore extends RM
@Override
public RMState loadState() throws Exception {
- return null;
+ throw new UnsupportedOperationException("Cannot load state from null store");
}
@Override
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Wed Jan 23 18:51:24 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ClientToken;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -49,11 +50,11 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
@@ -82,7 +83,6 @@ public class RMAppImpl implements RMApp,
private final String queue;
private final String name;
private final ApplicationSubmissionContext submissionContext;
- private final String clientTokenStr;
private final Dispatcher dispatcher;
private final YarnScheduler scheduler;
private final ApplicationMasterService masterService;
@@ -213,9 +213,9 @@ public class RMAppImpl implements RMApp,
public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
Configuration config, String name, String user, String queue,
- ApplicationSubmissionContext submissionContext, String clientTokenStr,
- YarnScheduler scheduler, ApplicationMasterService masterService,
- long submitTime) {
+ ApplicationSubmissionContext submissionContext,
+ YarnScheduler scheduler,
+ ApplicationMasterService masterService, long submitTime) {
this.applicationId = applicationId;
this.name = name;
@@ -226,7 +226,6 @@ public class RMAppImpl implements RMApp,
this.user = user;
this.queue = queue;
this.submissionContext = submissionContext;
- this.clientTokenStr = clientTokenStr;
this.scheduler = scheduler;
this.masterService = masterService;
this.submitTime = submitTime;
@@ -402,7 +401,7 @@ public class RMAppImpl implements RMApp,
try {
ApplicationAttemptId currentApplicationAttemptId = null;
- String clientToken = UNAVAILABLE;
+ ClientToken clientToken = null;
String trackingUrl = UNAVAILABLE;
String host = UNAVAILABLE;
String origTrackingUrl = UNAVAILABLE;
@@ -541,9 +540,9 @@ public class RMAppImpl implements RMApp,
appAttemptId.setApplicationId(applicationId);
appAttemptId.setAttemptId(attempts.size() + 1);
- RMAppAttempt attempt = new RMAppAttemptImpl(appAttemptId,
- clientTokenStr, rmContext, scheduler, masterService,
- submissionContext, conf);
+ RMAppAttempt attempt =
+ new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
+ submissionContext, conf);
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
if(startAttempt) {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Wed Jan 23 18:51:24 2013
@@ -23,10 +23,11 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ClientToken;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -91,7 +92,7 @@ public interface RMAppAttempt extends Ev
* The token required by the clients to talk to the application attempt
* @return the token required by the clients to talk to the application attempt
*/
- String getClientToken();
+ ClientToken getClientToken();
/**
* Diagnostics information for the application attempt.
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Wed Jan 23 18:51:24 2013
@@ -39,14 +39,17 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ClientToken;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -55,6 +58,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
@@ -119,7 +123,7 @@ public class RMAppAttemptImpl implements
private final WriteLock writeLock;
private final ApplicationAttemptId applicationAttemptId;
- private final String clientToken;
+ private ClientToken clientToken;
private final ApplicationSubmissionContext submissionContext;
//nodes on while this attempt's containers ran
@@ -347,11 +351,10 @@ public class RMAppAttemptImpl implements
.installTopology();
public RMAppAttemptImpl(ApplicationAttemptId appAttemptId,
- String clientToken, RMContext rmContext, YarnScheduler scheduler,
+ RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
Configuration conf) {
-
this.conf = conf;
this.applicationAttemptId = appAttemptId;
this.rmContext = rmContext;
@@ -359,7 +362,19 @@ public class RMAppAttemptImpl implements
this.submissionContext = submissionContext;
this.scheduler = scheduler;
this.masterService = masterService;
- this.clientToken = clientToken;
+
+ if (UserGroupInformation.isSecurityEnabled()) {
+
+ this.rmContext.getClientToAMTokenSecretManager().registerApplication(
+ appAttemptId);
+
+ Token<ClientTokenIdentifier> token =
+ new Token<ClientTokenIdentifier>(new ClientTokenIdentifier(
+ appAttemptId), this.rmContext.getClientToAMTokenSecretManager());
+ this.clientToken =
+ BuilderUtils.newClientToken(token.getIdentifier(), token.getKind()
+ .toString(), token.getPassword(), token.getService().toString());
+ }
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
@@ -477,7 +492,7 @@ public class RMAppAttemptImpl implements
}
@Override
- public String getClientToken() {
+ public ClientToken getClientToken() {
return this.clientToken;
}
@@ -963,6 +978,13 @@ public class RMAppAttemptImpl implements
appAttempt.rmContext.getAMFinishingMonitor().unregister(
appAttempt.getAppAttemptId());
+
+ // Unregister from the ClientTokenSecretManager
+ if (UserGroupInformation.isSecurityEnabled()) {
+ appAttempt.rmContext.getClientToAMTokenSecretManager()
+ .unRegisterApplication(appAttempt.getAppAttemptId());
+ }
+
if(!appAttempt.submissionContext.getUnmanagedAM()) {
// Tell the launcher to cleanup.
appAttempt.eventHandler.handle(new AMLauncherEvent(
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Wed Jan 23 18:51:24 2013
@@ -275,7 +275,7 @@ public class AppSchedulable extends Sche
// The desired container won't fit here, so reserve
reserve(application, priority, node, container, reserved);
- return Resources.none();
+ return FairScheduler.CONTAINER_RESERVED;
}
}
@@ -307,20 +307,27 @@ public class AppSchedulable extends Sche
// (not scheduled) in order to promote better locality.
synchronized (app) {
for (Priority priority : prioritiesToTry) {
+ if (app.getTotalRequiredResources(priority) <= 0) {
+ continue;
+ }
+
app.addSchedulingOpportunity(priority);
- NodeType allowedLocality = app.getAllowedLocalityLevel(priority,
- scheduler.getNumClusterNodes(), scheduler.getNodeLocalityThreshold(),
- scheduler.getRackLocalityThreshold());
+ ResourceRequest rackLocalRequest = app.getResourceRequest(priority,
+ node.getRackName());
ResourceRequest localRequest = app.getResourceRequest(priority,
node.getHostName());
- if (localRequest != null && localRequest.getNumContainers() != 0) {
+
+ NodeType allowedLocality = app.getAllowedLocalityLevel(priority,
+ scheduler.getNumClusterNodes(), scheduler.getNodeLocalityThreshold(),
+ scheduler.getRackLocalityThreshold());
+
+ if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
+ && localRequest != null && localRequest.getNumContainers() != 0) {
return assignContainer(node, app, priority,
localRequest, NodeType.NODE_LOCAL, reserved);
}
- ResourceRequest rackLocalRequest = app.getResourceRequest(priority,
- node.getRackName());
if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
&& (allowedLocality.equals(NodeType.RACK_LOCAL) ||
allowedLocality.equals(NodeType.OFF_SWITCH))) {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Wed Jan 23 18:51:24 2013
@@ -177,7 +177,10 @@ public class FSLeafQueue extends FSQueue
Collections.sort(appScheds, comparator);
for (AppSchedulable sched: appScheds) {
if (sched.getRunnable()) {
- return sched.assignContainer(node, reserved);
+ Resource assignedResource = sched.assignContainer(node, reserved);
+ if (!assignedResource.equals(Resources.none())) {
+ return assignedResource;
+ }
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Wed Jan 23 18:51:24 2013
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRejectedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
@@ -109,6 +110,10 @@ public class FairScheduler implements Re
private Clock clock;
private static final Log LOG = LogFactory.getLog(FairScheduler.class);
+
+ // Value that container assignment methods return when a container is
+ // reserved
+ public static final Resource CONTAINER_RESERVED = Resources.createResource(-1);
// How often fair shares are re-calculated (ms)
protected long UPDATE_INTERVAL = 500;
@@ -498,8 +503,11 @@ public class FairScheduler implements Re
// Enforce ACLs
UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)) {
- LOG.info("User " + userUgi.getUserName() +
- " cannot submit applications to queue " + queue.getName());
+ String msg = "User " + userUgi.getUserName() +
+ " cannot submit applications to queue " + queue.getName();
+ LOG.info(msg);
+ rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppAttemptRejectedEvent(applicationAttemptId, msg));
return;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java Wed Jan 23 18:51:24 2013
@@ -56,7 +56,7 @@ public class FairSchedulerConfiguration
/** Whether to assign multiple containers in one check-in. */
protected static final String ASSIGN_MULTIPLE = CONF_PREFIX + "assignmultiple";
- protected static final boolean DEFAULT_ASSIGN_MULTIPLE = true;
+ protected static final boolean DEFAULT_ASSIGN_MULTIPLE = false;
/** Whether to give more weight to apps requiring many resources. */
protected static final String SIZE_BASED_WEIGHT = CONF_PREFIX + "sizebasedweight";
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java Wed Jan 23 18:51:24 2013
@@ -23,26 +23,29 @@ import java.util.Map;
import javax.crypto.SecretKey;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager;
public class ClientToAMTokenSecretManagerInRM extends
BaseClientToAMTokenSecretManager {
// Per application master-keys for managing client-tokens
- private Map<ApplicationId, SecretKey> masterKeys =
- new HashMap<ApplicationId, SecretKey>();
+ private Map<ApplicationAttemptId, SecretKey> masterKeys =
+ new HashMap<ApplicationAttemptId, SecretKey>();
- public synchronized void registerApplication(ApplicationId applicationID) {
- this.masterKeys.put(applicationID, generateSecret());
+ public synchronized void registerApplication(
+ ApplicationAttemptId applicationAttemptID) {
+ this.masterKeys.put(applicationAttemptID, generateSecret());
}
- public synchronized void unRegisterApplication(ApplicationId applicationID) {
- this.masterKeys.remove(applicationID);
+ public synchronized void unRegisterApplication(
+ ApplicationAttemptId applicationAttemptID) {
+ this.masterKeys.remove(applicationAttemptID);
}
@Override
- public synchronized SecretKey getMasterKey(ApplicationId applicationID) {
- return this.masterKeys.get(applicationID);
+ public synchronized SecretKey getMasterKey(
+ ApplicationAttemptId applicationAttemptID) {
+ return this.masterKeys.get(applicationAttemptID);
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Wed Jan 23 18:51:24 2013
@@ -338,7 +338,7 @@ public class TestApplicationACLs {
Assert.assertEquals("Enemy should not see app rpc port!",
-1, appReport.getRpcPort());
Assert.assertEquals("Enemy should not see app client token!",
- UNAVAILABLE, appReport.getClientToken());
+ null, appReport.getClientToken());
Assert.assertEquals("Enemy should not see app diagnostics!",
UNAVAILABLE, appReport.getDiagnostics());
Assert.assertEquals("Enemy should not see app tracking url!",
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Wed Jan 23 18:51:24 2013
@@ -18,10 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -276,7 +276,7 @@ public class TestClientRMService {
private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler,
ApplicationId applicationId3, YarnConfiguration config, String queueName) {
return new RMAppImpl(applicationId3, rmContext, config, null, null,
- queueName, null, null, yarnScheduler, null, System
+ queueName, null, yarnScheduler, null , System
.currentTimeMillis());
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Wed Jan 23 18:51:24 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ClientToken;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -77,7 +78,7 @@ public abstract class MockAsm extends Mo
}
@Override
- public String getClientToken() {
+ public ClientToken getClientToken() {
throw new UnsupportedOperationException("Not supported yet.");
}
@@ -127,7 +128,7 @@ public abstract class MockAsm extends Mo
}
@Override
- public void setClientToken(String clientToken) {
+ public void setClientToken(ClientToken clientToken) {
throw new UnsupportedOperationException("Not supported yet.");
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Wed Jan 23 18:51:24 2013
@@ -168,7 +168,6 @@ public class TestRMAppTransitions {
Configuration conf = new YarnConfiguration();
// ensure max retries set to known value
conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, maxRetries);
- String clientTokenStr = "bogusstring";
YarnScheduler scheduler = mock(YarnScheduler.class);
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
@@ -177,11 +176,10 @@ public class TestRMAppTransitions {
submissionContext = new ApplicationSubmissionContextPBImpl();
}
- RMApp application = new RMAppImpl(applicationId, rmContext,
- conf, name, user,
- queue, submissionContext, clientTokenStr,
- scheduler,
- masterService, System.currentTimeMillis());
+ RMApp application =
+ new RMAppImpl(applicationId, rmContext, conf, name, user, queue,
+ submissionContext, scheduler, masterService,
+ System.currentTimeMillis());
testAppStartState(applicationId, user, name, queue, application);
return application;
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Wed Jan 23 18:51:24 2013
@@ -209,9 +209,9 @@ public class TestRMAppAttemptTransitions
unmanagedAM = false;
application = mock(RMApp.class);
- applicationAttempt =
- new RMAppAttemptImpl(applicationAttemptId, null, rmContext, scheduler,
- masterService, submissionContext, new Configuration());
+ applicationAttempt =
+ new RMAppAttemptImpl(applicationAttemptId, rmContext, scheduler,
+ masterService, submissionContext, new Configuration());
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Wed Jan 23 18:51:24 2013
@@ -41,20 +41,29 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -1275,4 +1284,136 @@ public class TestFairScheduler {
FSSchedulerApp app2 = scheduler.applications.get(attId2);
assertNull("The application was allowed", app2);
}
+
+ @Test
+ public void testMultipleNodesSingleRackRequest() throws Exception {
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
+ RMNode node3 = MockNodes.newNodeInfo(2, Resources.createResource(1024));
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+ NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
+ scheduler.handle(nodeEvent2);
+
+ ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
+ scheduler.addApplication(appId, "queue1", "user1");
+
+ // 1 request with 2 nodes on the same rack. another request with 1 node on
+ // a different rack
+ List<ResourceRequest> asks = new ArrayList<ResourceRequest>();
+ asks.add(createResourceRequest(1024, node1.getHostName(), 1, 1));
+ asks.add(createResourceRequest(1024, node2.getHostName(), 1, 1));
+ asks.add(createResourceRequest(1024, node3.getHostName(), 1, 1));
+ asks.add(createResourceRequest(1024, node1.getRackName(), 1, 1));
+ asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1));
+ asks.add(createResourceRequest(1024, RMNode.ANY, 1, 2));
+
+ scheduler.allocate(appId, asks, new ArrayList<ContainerId>());
+
+ // node 1 checks in
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent1 = new NodeUpdateSchedulerEvent(node1,
+ new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
+ scheduler.handle(updateEvent1);
+ // should assign node local
+ assertEquals(1, scheduler.applications.get(appId).getLiveContainers().size());
+
+ // node 2 checks in
+ scheduler.update();
+ NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2,
+ new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
+ scheduler.handle(updateEvent2);
+ // should assign rack local
+ assertEquals(2, scheduler.applications.get(appId).getLiveContainers().size());
+ }
+
+ @Test
+ public void testFifoWithinQueue() throws Exception {
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(3072));
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Even if submitted at exact same time, apps will be deterministically
+ // ordered by name.
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1",
+ "user1", 2);
+ ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1",
+ "user1", 2);
+ FSSchedulerApp app1 = scheduler.applications.get(attId1);
+ FSSchedulerApp app2 = scheduler.applications.get(attId2);
+
+ FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1");
+ queue1.setSchedulingMode(SchedulingMode.FIFO);
+
+ scheduler.update();
+
+ // First two containers should go to app 1, third should go to app 2.
+ // Because tests set assignmultiple to false, each heartbeat assigns a single
+ // container.
+
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1,
+ new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
+
+ scheduler.handle(updateEvent);
+ assertEquals(1, app1.getLiveContainers().size());
+ assertEquals(0, app2.getLiveContainers().size());
+
+ scheduler.handle(updateEvent);
+ assertEquals(2, app1.getLiveContainers().size());
+ assertEquals(0, app2.getLiveContainers().size());
+
+ scheduler.handle(updateEvent);
+ assertEquals(2, app1.getLiveContainers().size());
+ assertEquals(1, app2.getLiveContainers().size());
+ }
+
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testNotAllowSubmitApplication() throws Exception {
+ // Set acl's
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queue1\">");
+ out.println("<aclSubmitApps>userallow</aclSubmitApps>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ int appId = this.APP_ID++;
+ String user = "usernotallow";
+ String queue = "queue1";
+ ApplicationId applicationId = MockApps.newAppID(appId);
+ String name = MockApps.newAppName();
+ ApplicationMasterService masterService =
+ new ApplicationMasterService(resourceManager.getRMContext(), scheduler);
+ ApplicationSubmissionContext submissionContext = new ApplicationSubmissionContextPBImpl();
+ RMApp application =
+ new RMAppImpl(applicationId, resourceManager.getRMContext(), conf, name, user,
+ queue, submissionContext, scheduler, masterService,
+ System.currentTimeMillis());
+ resourceManager.getRMContext().getRMApps().putIfAbsent(applicationId, application);
+ application.handle(new RMAppEvent(applicationId, RMAppEventType.START));
+
+ ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
+ attId.setAttemptId(this.ATTEMPT_ID++);
+ attId.setApplicationId(applicationId);
+ scheduler.addApplication(attId, queue, user);
+
+ final int MAX_TRIES=20;
+ int numTries = 0;
+ while (application.getFinishTime() == 0 && numTries < MAX_TRIES) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException ex) {ex.printStackTrace();}
+ numTries++;
+ }
+ assertEquals(FinalApplicationStatus.FAILED, application.getFinalApplicationStatus());
+ }
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java Wed Jan 23 18:51:24 2013
@@ -52,8 +52,10 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ClientToken;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -67,6 +69,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.ProtoUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
@@ -106,14 +109,14 @@ public class TestClientTokens {
private static class CustomAM extends AbstractService implements
CustomProtocol {
- private final ApplicationId appId;
+ private final ApplicationAttemptId appAttemptId;
private final String secretKey;
private InetSocketAddress address;
private boolean pinged = false;
- public CustomAM(ApplicationId appId, String secretKeyStr) {
+ public CustomAM(ApplicationAttemptId appId, String secretKeyStr) {
super("CustomAM");
- this.appId = appId;
+ this.appAttemptId = appId;
this.secretKey = secretKeyStr;
}
@@ -128,7 +131,7 @@ public class TestClientTokens {
ClientToAMTokenSecretManager secretManager = null;
byte[] bytes = Base64.decodeBase64(this.secretKey);
- secretManager = new ClientToAMTokenSecretManager(this.appId, bytes);
+ secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, bytes);
Server server;
try {
server =
@@ -216,7 +219,7 @@ public class TestClientTokens {
GetApplicationReportResponse reportResponse =
rm.getClientRMService().getApplicationReport(request);
ApplicationReport appReport = reportResponse.getApplicationReport();
- String clientTokenEncoded = appReport.getClientToken();
+ ClientToken clientToken = appReport.getClientToken();
// Wait till AM is 'launched'
int waitTime = 0;
@@ -226,9 +229,11 @@ public class TestClientTokens {
Assert.assertNotNull(containerManager.clientTokensSecret);
// Start the AM with the correct shared-secret.
+ ApplicationAttemptId appAttemptId =
+ app.getAppAttempts().keySet().iterator().next();
+ Assert.assertNotNull(appAttemptId);
final CustomAM am =
- new CustomAM(app.getApplicationId(),
- containerManager.clientTokensSecret);
+ new CustomAM(appAttemptId, containerManager.clientTokensSecret);
am.init(conf);
am.start();
@@ -249,21 +254,19 @@ public class TestClientTokens {
// Verify denial for a malicious user
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
- Token<ClientTokenIdentifier> clientToken =
- new Token<ClientTokenIdentifier>();
- clientToken.decodeFromUrlString(clientTokenEncoded);
- // RPC layer client expects ip:port as service for tokens
- SecurityUtil.setTokenService(clientToken, am.address);
+ Token<ClientTokenIdentifier> token =
+ ProtoUtils.convertFromProtoFormat(clientToken, am.address);
// Malicious user, messes with appId
ClientTokenIdentifier maliciousID =
- new ClientTokenIdentifier(BuilderUtils.newApplicationId(app
- .getApplicationId().getClusterTimestamp(), 42));
+ new ClientTokenIdentifier(BuilderUtils.newApplicationAttemptId(
+ BuilderUtils.newApplicationId(app.getApplicationId()
+ .getClusterTimestamp(), 42), 43));
Token<ClientTokenIdentifier> maliciousToken =
new Token<ClientTokenIdentifier>(maliciousID.getBytes(),
- clientToken.getPassword(), clientToken.getKind(),
- clientToken.getService());
+ token.getPassword(), token.getKind(),
+ token.getService());
ugi.addToken(maliciousToken);
try {
@@ -297,7 +300,7 @@ public class TestClientTokens {
// Now for an authenticated user
ugi = UserGroupInformation.createRemoteUser("me");
- ugi.addToken(clientToken);
+ ugi.addToken(token);
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1437623&r1=1437622&r2=1437623&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Wed Jan 23 18:51:24 2013
@@ -132,20 +132,45 @@ Hadoop MapReduce Next Generation - Fair
* Whether to use the username associated with the allocation as the default
queue name, in the event that a queue name is not specified. If this is set
to "false" or unset, all jobs have a shared default queue, called "default".
+ Defaults to true.
* <<<yarn.scheduler.fair.preemption>>>
* Whether to use preemption. Note that preemption is experimental in the current
- version.
+ version. Defaults to false.
* <<<yarn.scheduler.fair.sizebasedweight>>>
* Whether to assign shares to individual apps based on their size, rather than
- providing an equal share to all apps regardless of size.
+ providing an equal share to all apps regardless of size. Defaults to false.
* <<<yarn.scheduler.fair.assignmultiple>>>
- * Whether to allow multiple container assignments in one heartbeat.
+ * Whether to allow multiple container assignments in one heartbeat. Defaults
+ to false.
+
+ * <<<yarn.scheduler.fair.max.assign>>>
+
+ * If assignmultiple is true, the maximum amount of containers that can be
+ assigned in one heartbeat. Defaults to -1, which sets no limit.
+
+ * <<<locality.threshold.node>>>
+
+ * For applications that request containers on particular nodes, the number of
+ scheduling opportunities since the last container assignment to wait before
+ accepting a placement on another node. Expressed as a float between 0 and 1,
+ which, as a fraction of the cluster size, is the number of scheduling
+ opportunities to pass up. The default value of -1.0 means don't pass up any
+ scheduling opportunities.
+
+ * <<<locality.threshold.rack>>>
+
+ * For applications that request containers on particular racks, the number of
+ scheduling opportunities since the last container assignment to wait before
+ accepting a placement on another rack. Expressed as a float between 0 and 1,
+ which, as a fraction of the cluster size, is the number of scheduling
+ opportunities to pass up. The default value of -1.0 means don't pass up any
+ scheduling opportunities.
Allocation file format
@@ -166,6 +191,14 @@ Allocation file format
* schedulingMode: either "fifo" or "fair" depending on the in-queue scheduling
policy desired
+ * aclSubmitApps: a list of users that can submit apps to the queue. A (default)
+ value of "*" means that any users can submit apps. A queue inherits the ACL of
+ its parent, so if a queue2 descends from queue1, and user1 is in queue1's ACL,
+ and user2 is in queue2's ACL, then both users may submit to queue2.
+
+ * minSharePreemptionTimeout: number of seconds the queue is under its minimum share
+ before it will try to preempt containers to take resources from other queues.
+
* <<User elements>>, which represent settings governing the behavior of individual
users. They can contain a single property: maxRunningApps, a limit on the
number of running apps for a particular user.
@@ -173,6 +206,10 @@ Allocation file format
* <<A userMaxAppsDefault element>>, which sets the default running app limit
for any users whose limit is not otherwise specified.
+ * <<A fairSharePreemptionTimeout element>>, number of seconds a queue is under
+ its fair share before it will try to preempt containers to take resources from
+ other queues.
+
An example allocation file is given here:
---