You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by wa...@apache.org on 2013/09/20 01:42:29 UTC
svn commit: r1524865 [6/8] - in
/hadoop/common/branches/HDFS-4949/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protoc...
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java Thu Sep 19 23:42:10 2013
@@ -65,7 +65,11 @@ public class MemoryRMStateStore extends
@Override
public synchronized void initInternal(Configuration conf) {
}
-
+
+ @Override
+ protected synchronized void startInternal() throws Exception {
+ }
+
@Override
protected synchronized void closeInternal() throws Exception {
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java Thu Sep 19 23:42:10 2013
@@ -21,10 +21,10 @@ package org.apache.hadoop.yarn.server.re
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@Unstable
public class NullRMStateStore extends RMStateStore {
@@ -35,6 +35,11 @@ public class NullRMStateStore extends RM
}
@Override
+ protected void startInternal() throws Exception {
+ // Do nothing
+ }
+
+ @Override
protected void closeInternal() throws Exception {
// Do nothing
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Thu Sep 19 23:42:10 2013
@@ -24,6 +24,8 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import javax.crypto.SecretKey;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -34,6 +36,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -43,7 +46,6 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@@ -60,23 +62,28 @@ import org.apache.hadoop.yarn.server.res
* Real store implementations need to derive from it and implement blocking
* store and load methods to actually store and load the state.
*/
-public abstract class RMStateStore {
+public abstract class RMStateStore extends AbstractService {
+
public static final Log LOG = LogFactory.getLog(RMStateStore.class);
+ public RMStateStore() {
+ super(RMStateStore.class.getName());
+ }
+
/**
* State of an application attempt
*/
public static class ApplicationAttemptState {
final ApplicationAttemptId attemptId;
final Container masterContainer;
- final Credentials appAttemptTokens;
+ final Credentials appAttemptCredentials;
public ApplicationAttemptState(ApplicationAttemptId attemptId,
Container masterContainer,
- Credentials appAttemptTokens) {
+ Credentials appAttemptCredentials) {
this.attemptId = attemptId;
this.masterContainer = masterContainer;
- this.appAttemptTokens = appAttemptTokens;
+ this.appAttemptCredentials = appAttemptCredentials;
}
public Container getMasterContainer() {
@@ -85,8 +92,8 @@ public abstract class RMStateStore {
public ApplicationAttemptId getAttemptId() {
return attemptId;
}
- public Credentials getAppAttemptTokens() {
- return appAttemptTokens;
+ public Credentials getAppAttemptCredentials() {
+ return appAttemptCredentials;
}
}
@@ -174,31 +181,39 @@ public abstract class RMStateStore {
* Dispatcher used to send state operation completion events to
* ResourceManager services
*/
- public void setDispatcher(Dispatcher dispatcher) {
+ public void setRMDispatcher(Dispatcher dispatcher) {
this.rmDispatcher = dispatcher;
}
AsyncDispatcher dispatcher;
- public synchronized void init(Configuration conf) throws Exception{
+ public synchronized void serviceInit(Configuration conf) throws Exception{
// create async handler
dispatcher = new AsyncDispatcher();
dispatcher.init(conf);
dispatcher.register(RMStateStoreEventType.class,
new ForwardingEventHandler());
- dispatcher.start();
-
initInternal(conf);
}
+
+ protected synchronized void serviceStart() throws Exception {
+ dispatcher.start();
+ startInternal();
+ }
/**
* Derived classes initialize themselves using this method.
- * The base class is initialized and the event dispatcher is ready to use at
- * this point
*/
protected abstract void initInternal(Configuration conf) throws Exception;
-
- public synchronized void close() throws Exception {
+
+ /**
+ * Derived classes start themselves using this method.
+ * The base class is started and the event dispatcher is ready to use at
+ * this point
+ */
+ protected abstract void startInternal() throws Exception;
+
+ public synchronized void serviceStop() throws Exception {
closeInternal();
dispatcher.stop();
}
@@ -251,7 +266,7 @@ public abstract class RMStateStore {
* RMAppAttemptStoredEvent will be sent on completion to notify the RMAppAttempt
*/
public synchronized void storeApplicationAttempt(RMAppAttempt appAttempt) {
- Credentials credentials = getTokensFromAppAttempt(appAttempt);
+ Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
ApplicationAttemptState attemptState =
new ApplicationAttemptState(appAttempt.getAppAttemptId(),
@@ -351,7 +366,7 @@ public abstract class RMStateStore {
app.getSubmitTime(), app.getApplicationSubmissionContext(),
app.getUser());
for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
- Credentials credentials = getTokensFromAppAttempt(appAttempt);
+ Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
ApplicationAttemptState attemptState =
new ApplicationAttemptState(appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(), credentials);
@@ -381,17 +396,21 @@ public abstract class RMStateStore {
// YARN-986
public static final Text AM_RM_TOKEN_SERVICE = new Text(
"AM_RM_TOKEN_SERVICE");
+
+ public static final Text AM_CLIENT_TOKEN_MASTER_KEY_NAME =
+ new Text("YARN_CLIENT_TOKEN_MASTER_KEY");
- private Credentials getTokensFromAppAttempt(RMAppAttempt appAttempt) {
+ private Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
Credentials credentials = new Credentials();
Token<AMRMTokenIdentifier> appToken = appAttempt.getAMRMToken();
if(appToken != null){
credentials.addToken(AM_RM_TOKEN_SERVICE, appToken);
}
- Token<ClientToAMTokenIdentifier> clientToAMToken =
- appAttempt.getClientToAMToken();
- if(clientToAMToken != null){
- credentials.addToken(clientToAMToken.getService(), clientToAMToken);
+ SecretKey clientTokenMasterKey =
+ appAttempt.getClientTokenMasterKey();
+ if(clientTokenMasterKey != null){
+ credentials.addSecretKey(AM_CLIENT_TOKEN_MASTER_KEY_NAME,
+ clientTokenMasterKey.getEncoded());
}
return credentials;
}
@@ -431,7 +450,7 @@ public abstract class RMStateStore {
((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
Exception storedException = null;
- Credentials credentials = attemptState.getAppAttemptTokens();
+ Credentials credentials = attemptState.getAppAttemptCredentials();
ByteBuffer appAttemptTokens = null;
try {
if(credentials != null){
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java Thu Sep 19 23:42:10 2013
@@ -128,10 +128,12 @@ public interface RMApp extends EventHand
* <li>resource usage report - all values are -1</li>
* </ul>
*
+ * @param clientUserName the user name of the client requesting the report
* @param allowAccess whether to allow full access to the report
* @return the {@link ApplicationReport} detailing the status of the application.
*/
- ApplicationReport createAndGetApplicationReport(boolean allowAccess);
+ ApplicationReport createAndGetApplicationReport(String clientUserName,
+ boolean allowAccess);
/**
* To receive the collection of all {@link RMNode}s whose updates have been
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Thu Sep 19 23:42:10 2013
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
-import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
@@ -44,7 +43,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -55,6 +53,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@@ -378,29 +377,6 @@ public class RMAppImpl implements RMApp,
}
}
- private YarnApplicationState createApplicationState(RMAppState rmAppState) {
- switch(rmAppState) {
- case NEW:
- return YarnApplicationState.NEW;
- case NEW_SAVING:
- return YarnApplicationState.NEW_SAVING;
- case SUBMITTED:
- return YarnApplicationState.SUBMITTED;
- case ACCEPTED:
- return YarnApplicationState.ACCEPTED;
- case RUNNING:
- return YarnApplicationState.RUNNING;
- case FINISHING:
- case FINISHED:
- return YarnApplicationState.FINISHED;
- case KILLED:
- return YarnApplicationState.KILLED;
- case FAILED:
- return YarnApplicationState.FAILED;
- }
- throw new YarnRuntimeException("Unknown state passed!");
- }
-
private FinalApplicationStatus createFinalApplicationStatus(RMAppState state) {
switch(state) {
case NEW:
@@ -434,7 +410,8 @@ public class RMAppImpl implements RMApp,
}
@Override
- public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+ public ApplicationReport createAndGetApplicationReport(String clientUserName,
+ boolean allowAccess) {
this.readLock.lock();
try {
@@ -455,15 +432,18 @@ public class RMAppImpl implements RMApp,
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
trackingUrl = this.currentAttempt.getTrackingUrl();
origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
- Token<ClientToAMTokenIdentifier> attemptClientToAMToken =
- this.currentAttempt.getClientToAMToken();
- if (attemptClientToAMToken != null) {
- clientToAMToken =
- BuilderUtils.newClientToAMToken(
- attemptClientToAMToken.getIdentifier(),
- attemptClientToAMToken.getKind().toString(),
- attemptClientToAMToken.getPassword(),
- attemptClientToAMToken.getService().toString());
+ if (UserGroupInformation.isSecurityEnabled()) {
+ // get a token so the client can communicate with the app attempt
+ // NOTE: token may be unavailable if the attempt is not running
+ Token<ClientToAMTokenIdentifier> attemptClientToAMToken =
+ this.currentAttempt.createClientToken(clientUserName);
+ if (attemptClientToAMToken != null) {
+ clientToAMToken = BuilderUtils.newClientToAMToken(
+ attemptClientToAMToken.getIdentifier(),
+ attemptClientToAMToken.getKind().toString(),
+ attemptClientToAMToken.getPassword(),
+ attemptClientToAMToken.getService().toString());
+ }
}
host = this.currentAttempt.getHost();
rpcPort = this.currentAttempt.getRpcPort();
@@ -474,20 +454,15 @@ public class RMAppImpl implements RMApp,
if (currentAttempt != null &&
currentAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
- try {
- if (getApplicationSubmissionContext().getUnmanagedAM() &&
- getUser().equals(UserGroupInformation.getCurrentUser().getUserName())) {
- Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
- if (token != null) {
- amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(),
- token.getKind().toString(), token.getPassword(),
- token.getService().toString());
- }
+ if (getApplicationSubmissionContext().getUnmanagedAM() &&
+ clientUserName != null && getUser().equals(clientUserName)) {
+ Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
+ if (token != null) {
+ amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(),
+ token.getKind().toString(), token.getPassword(),
+ token.getService().toString());
}
- } catch (IOException ex) {
- LOG.warn("UserGroupInformation.getCurrentUser() error: " +
- ex.toString(), ex);
- }
+ }
}
}
@@ -500,7 +475,7 @@ public class RMAppImpl implements RMApp,
return BuilderUtils.newApplicationReport(this.applicationId,
currentApplicationAttemptId, this.user, this.queue,
this.name, host, rpcPort, clientToAMToken,
- createApplicationState(this.stateMachine.getCurrentState()), diags,
+ RMServerUtils.createApplicationState(this.stateMachine.getCurrentState()), diags,
trackingUrl, this.startTime, this.finishTime, finishState,
appUsageReport, origTrackingUrl, progress, this.applicationType,
amrmToken);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Thu Sep 19 23:42:10 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.re
import java.util.List;
import java.util.Set;
+import javax.crypto.SecretKey;
+
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -91,12 +93,6 @@ public interface RMAppAttempt extends Ev
String getWebProxyBase();
/**
- * The token required by the clients to talk to the application attempt
- * @return the token required by the clients to talk to the application attempt
- */
- Token<ClientToAMTokenIdentifier> getClientToAMToken();
-
- /**
* Diagnostics information for the application attempt.
* @return diagnostics information for the application attempt.
*/
@@ -155,6 +151,19 @@ public interface RMAppAttempt extends Ev
Token<AMRMTokenIdentifier> getAMRMToken();
/**
+ * The master key for client-to-AM tokens for this app attempt
+ * @return The master key for client-to-AM tokens for this app attempt
+ */
+ SecretKey getClientTokenMasterKey();
+
+ /**
+ * Create a token for authenticating a client connection to the app attempt
+ * @param clientName the name of the client requesting the token
+ * @return the token or null if the attempt is not running
+ */
+ Token<ClientToAMTokenIdentifier> createClientToken(String clientName);
+
+ /**
* Get application container and resource usage information.
* @return an ApplicationResourceUsageReport object.
*/
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Thu Sep 19 23:42:10 2013
@@ -33,12 +33,13 @@ import java.util.concurrent.locks.Reentr
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import javax.crypto.SecretKey;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -61,7 +62,6 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
@@ -90,6 +90,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
@@ -126,9 +127,9 @@ public class RMAppAttemptImpl implements
private final WriteLock writeLock;
private final ApplicationAttemptId applicationAttemptId;
- private Token<ClientToAMTokenIdentifier> clientToAMToken;
private final ApplicationSubmissionContext submissionContext;
private Token<AMRMTokenIdentifier> amrmToken = null;
+ private SecretKey clientTokenMasterKey = null;
//nodes on while this attempt's containers ran
private final Set<NodeId> ranNodes =
@@ -441,7 +442,8 @@ public class RMAppAttemptImpl implements
public String getTrackingUrl() {
this.readLock.lock();
try {
- return this.proxiedTrackingUrl;
+ return (getSubmissionContext().getUnmanagedAM()) ?
+ this.origTrackingUrl : this.proxiedTrackingUrl;
} finally {
this.readLock.unlock();
}
@@ -499,8 +501,8 @@ public class RMAppAttemptImpl implements
}
@Override
- public Token<ClientToAMTokenIdentifier> getClientToAMToken() {
- return this.clientToAMToken;
+ public SecretKey getClientTokenMasterKey() {
+ return this.clientTokenMasterKey;
}
@Override
@@ -509,6 +511,26 @@ public class RMAppAttemptImpl implements
}
@Override
+ public Token<ClientToAMTokenIdentifier> createClientToken(String client) {
+ this.readLock.lock();
+
+ try {
+ Token<ClientToAMTokenIdentifier> token = null;
+ ClientToAMTokenSecretManagerInRM secretMgr =
+ this.rmContext.getClientToAMTokenSecretManager();
+ if (client != null &&
+ secretMgr.getMasterKey(this.applicationAttemptId) != null) {
+ token = new Token<ClientToAMTokenIdentifier>(
+ new ClientToAMTokenIdentifier(this.applicationAttemptId, client),
+ secretMgr);
+ }
+ return token;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
public String getDiagnostics() {
this.readLock.lock();
@@ -659,7 +681,7 @@ public class RMAppAttemptImpl implements
ApplicationAttemptState attemptState = appState.getAttempt(getAppAttemptId());
assert attemptState != null;
setMasterContainer(attemptState.getMasterContainer());
- recoverAppAttemptTokens(attemptState.getAppAttemptTokens());
+ recoverAppAttemptCredentials(attemptState.getAppAttemptCredentials());
LOG.info("Recovered attempt: AppId: " + getAppAttemptId().getApplicationId()
+ " AttemptId: " + getAppAttemptId()
+ " MasterContainer: " + masterContainer);
@@ -668,17 +690,16 @@ public class RMAppAttemptImpl implements
RMAppAttemptEventType.RECOVER));
}
- private void recoverAppAttemptTokens(Credentials appAttemptTokens) {
+ private void recoverAppAttemptCredentials(Credentials appAttemptTokens) {
if (appAttemptTokens == null) {
return;
}
- if (UserGroupInformation.isSecurityEnabled()) {
- ClientToAMTokenSelector clientToAMTokenSelector =
- new ClientToAMTokenSelector();
- this.clientToAMToken =
- clientToAMTokenSelector.selectToken(new Text(),
- appAttemptTokens.getAllTokens());
+ if (UserGroupInformation.isSecurityEnabled()) {
+ byte[] clientTokenMasterKeyBytes = appAttemptTokens.getSecretKey(
+ RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME);
+ clientTokenMasterKey = rmContext.getClientToAMTokenSecretManager()
+ .registerMasterKey(applicationAttemptId, clientTokenMasterKeyBytes);
}
// Only one AMRMToken is stored per-attempt, so this should be fine. Can't
@@ -715,15 +736,9 @@ public class RMAppAttemptImpl implements
.registerAppAttempt(appAttempt.applicationAttemptId);
if (UserGroupInformation.isSecurityEnabled()) {
-
- appAttempt.rmContext.getClientToAMTokenSecretManager()
- .registerApplication(appAttempt.applicationAttemptId);
-
- // create clientToAMToken
- appAttempt.clientToAMToken =
- new Token<ClientToAMTokenIdentifier>(new ClientToAMTokenIdentifier(
- appAttempt.applicationAttemptId),
- appAttempt.rmContext.getClientToAMTokenSecretManager());
+ appAttempt.clientTokenMasterKey = appAttempt.rmContext
+ .getClientToAMTokenSecretManager()
+ .registerApplication(appAttempt.applicationAttemptId);
}
// create AMRMToken
@@ -761,6 +776,8 @@ public class RMAppAttemptImpl implements
rejectedEvent.getApplicationAttemptId().getApplicationId(),
message)
);
+
+ appAttempt.removeCredentials(appAttempt);
}
}
@@ -816,7 +833,11 @@ public class RMAppAttemptImpl implements
Allocation amContainerAllocation = appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId, EMPTY_CONTAINER_REQUEST_LIST,
EMPTY_CONTAINER_RELEASE_LIST, null, null);
-
+ // There must be at least one container allocated, because a
+ // CONTAINER_ALLOCATED is emitted after an RMContainer is constructed,
+ // and is put in SchedulerApplication#newlyAllocatedContainers. Then,
+ // YarnScheduler#allocate will fetch it.
+ assert amContainerAllocation.getContainers().size() != 0;
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
0));
@@ -847,7 +868,6 @@ public class RMAppAttemptImpl implements
@Override
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
-
ApplicationAttemptId appAttemptId = appAttempt.getAppAttemptId();
// Tell the AMS. Unregister from the ApplicationMasterService
@@ -894,9 +914,7 @@ public class RMAppAttemptImpl implements
appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId,
finalAttemptState));
- // Remove the AppAttempt from the AMRMTokenSecretManager
- appAttempt.rmContext.getAMRMTokenSecretManager()
- .applicationMasterFinished(appAttemptId);
+ appAttempt.removeCredentials(appAttempt);
}
}
@@ -970,7 +988,7 @@ public class RMAppAttemptImpl implements
}
}
- private static final class AMRegisteredTransition extends BaseTransition {
+ static final class AMRegisteredTransition extends BaseTransition {
@Override
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
@@ -1015,7 +1033,6 @@ public class RMAppAttemptImpl implements
" exitCode: " + status.getExitStatus() +
" due to: " + status.getDiagnostics() + "." +
"Failing this attempt.");
-
// Tell the app, scheduler
super.transition(appAttempt, finishEvent);
}
@@ -1042,12 +1059,6 @@ public class RMAppAttemptImpl implements
appAttempt.rmContext.getAMFinishingMonitor().unregister(
appAttempt.getAppAttemptId());
- // Unregister from the ClientToAMTokenSecretManager
- if (UserGroupInformation.isSecurityEnabled()) {
- appAttempt.rmContext.getClientToAMTokenSecretManager()
- .unRegisterApplication(appAttempt.getAppAttemptId());
- }
-
if(!appAttempt.submissionContext.getUnmanagedAM()) {
// Tell the launcher to cleanup.
appAttempt.eventHandler.handle(new AMLauncherEvent(
@@ -1116,10 +1127,6 @@ public class RMAppAttemptImpl implements
appAttempt.rmContext.getAMLivelinessMonitor().unregister(appAttemptId);
- // Remove the AppAttempt from the AMRMTokenSecretManager
- appAttempt.rmContext.getAMRMTokenSecretManager()
- .applicationMasterFinished(appAttemptId);
-
appAttempt.progress = 1.0f;
RMAppAttemptUnregistrationEvent unregisterEvent
@@ -1267,4 +1274,16 @@ public class RMAppAttemptImpl implements
+ " MasterContainer: " + masterContainer);
store.storeApplicationAttempt(this);
}
+
+ private void removeCredentials(RMAppAttemptImpl appAttempt) {
+ // Unregister from the ClientToAMTokenSecretManager
+ if (UserGroupInformation.isSecurityEnabled()) {
+ appAttempt.rmContext.getClientToAMTokenSecretManager()
+ .unRegisterApplication(appAttempt.getAppAttemptId());
+ }
+
+ // Remove the AppAttempt from the AMRMTokenSecretManager
+ appAttempt.rmContext.getAMRMTokenSecretManager()
+ .applicationMasterFinished(appAttempt.getAppAttemptId());
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java Thu Sep 19 23:42:10 2013
@@ -35,6 +35,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -392,9 +393,18 @@ public class RMNodeImpl implements RMNod
}
}
- private void updateMetricsForDeactivatedNode(NodeState finalState) {
+ private void updateMetricsForDeactivatedNode(NodeState initialState,
+ NodeState finalState) {
ClusterMetrics metrics = ClusterMetrics.getMetrics();
- metrics.decrNumActiveNodes();
+
+ switch (initialState) {
+ case RUNNING:
+ metrics.decrNumActiveNodes();
+ break;
+ case UNHEALTHY:
+ metrics.decrNumUnhealthyNMs();
+ break;
+ }
switch (finalState) {
case DECOMMISSIONED:
@@ -504,7 +514,8 @@ public class RMNodeImpl implements RMNod
// If the current state is NodeState.UNHEALTHY
// Then node is already been removed from the
// Scheduler
- if (!rmNode.getState().equals(NodeState.UNHEALTHY)) {
+ NodeState initialState = rmNode.getState();
+ if (!initialState.equals(NodeState.UNHEALTHY)) {
rmNode.context.getDispatcher().getEventHandler()
.handle(new NodeRemovedSchedulerEvent(rmNode));
}
@@ -519,7 +530,7 @@ public class RMNodeImpl implements RMNod
rmNode.context.getInactiveRMNodes().put(rmNode.nodeId.getHost(), rmNode);
//Update the metrics
- rmNode.updateMetricsForDeactivatedNode(finalState);
+ rmNode.updateMetricsForDeactivatedNode(initialState, finalState);
}
}
@@ -549,7 +560,8 @@ public class RMNodeImpl implements RMNod
new NodesListManagerEvent(
NodesListManagerEventType.NODE_UNUSABLE, rmNode));
// Update metrics
- rmNode.updateMetricsForDeactivatedNode(NodeState.UNHEALTHY);
+ rmNode.updateMetricsForDeactivatedNode(rmNode.getState(),
+ NodeState.UNHEALTHY);
return NodeState.UNHEALTHY;
}
@@ -601,9 +613,13 @@ public class RMNodeImpl implements RMNod
rmNode.context.getDispatcher().getEventHandler().handle(
new NodeUpdateSchedulerEvent(rmNode));
}
-
- rmNode.context.getDelegationTokenRenewer().updateKeepAliveApplications(
+
+ // Update DTRenewer in secure mode to keep these apps alive. Today this is
+ // needed for log-aggregation to finish long after the apps are gone.
+ if (UserGroupInformation.isSecurityEnabled()) {
+ rmNode.context.getDelegationTokenRenewer().updateKeepAliveApplications(
statusEvent.getKeepAliveAppIds());
+ }
return NodeState.RUNNING;
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java Thu Sep 19 23:42:10 2013
@@ -281,7 +281,7 @@ public class AppSchedulingInfo {
// Update future requirements
nodeLocalRequest.setNumContainers(nodeLocalRequest.getNumContainers() - 1);
if (nodeLocalRequest.getNumContainers() == 0) {
- this.requests.get(priority).remove(node.getHostName());
+ this.requests.get(priority).remove(node.getNodeName());
}
ResourceRequest rackLocalRequest = requests.get(priority).get(
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java Thu Sep 19 23:42:10 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
* Represents a YARN Cluster Node from the viewpoint of the scheduler.
@@ -30,10 +31,17 @@ import org.apache.hadoop.yarn.api.record
public abstract class SchedulerNode {
/**
- * Get hostname.
- * @return hostname
+ * Get the name of the node for scheduling matching decisions.
+ * <p/>
+ * Typically this is the 'hostname' reported by the node, but it could be
+ * configured to be 'hostname:port' reported by the node via the
+ * {@link YarnConfiguration#RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME} constant.
+ * The main usecase of this is Yarn minicluster to be able to differentiate
+ * node manager instances by their port number.
+ *
+ * @return name of the node for scheduling matching decisions.
*/
- public abstract String getHostName();
+ public abstract String getNodeName();
/**
* Get rackname.
@@ -59,4 +67,9 @@ public abstract class SchedulerNode {
*/
public abstract int getNumContainers();
+ /**
+ * Get total resources on the node.
+ * @return total resources on the node.
+ */
+ public abstract Resource getTotalResource();
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Thu Sep 19 23:42:10 2013
@@ -63,11 +63,11 @@ public class SchedulerUtils {
public static final String UNRESERVED_CONTAINER =
"Container reservation no longer required.";
-
+
/**
* Utility to create a {@link ContainerStatus} during exceptional
* circumstances.
- *
+ *
* @param containerId {@link ContainerId} of returned/released/lost container.
* @param diagnostics diagnostic message
* @return <code>ContainerStatus</code> for an returned/released/lost
@@ -75,12 +75,41 @@ public class SchedulerUtils {
*/
public static ContainerStatus createAbnormalContainerStatus(
ContainerId containerId, String diagnostics) {
+ return createAbnormalContainerStatus(containerId,
+ ContainerExitStatus.ABORTED, diagnostics);
+ }
+
+ /**
+ * Utility to create a {@link ContainerStatus} during exceptional
+ * circumstances.
+ *
+ * @param containerId {@link ContainerId} of returned/released/lost container.
+ * @param diagnostics diagnostic message
+ * @return <code>ContainerStatus</code> for an returned/released/lost
+ * container
+ */
+ public static ContainerStatus createPreemptedContainerStatus(
+ ContainerId containerId, String diagnostics) {
+ return createAbnormalContainerStatus(containerId,
+ ContainerExitStatus.PREEMPTED, diagnostics);
+ }
+
+ /**
+ * Utility to create a {@link ContainerStatus} during exceptional
+ * circumstances.
+ *
+ * @param containerId {@link ContainerId} of returned/released/lost container.
+ * @param diagnostics diagnostic message
+ * @return <code>ContainerStatus</code> for an returned/released/lost
+ * container
+ */
+ private static ContainerStatus createAbnormalContainerStatus(
+ ContainerId containerId, int exitStatus, String diagnostics) {
ContainerStatus containerStatus =
recordFactory.newRecordInstance(ContainerStatus.class);
containerStatus.setContainerId(containerId);
containerStatus.setDiagnostics(diagnostics);
- containerStatus.setExitStatus(
- ContainerExitStatus.ABORTED);
+ containerStatus.setExitStatus(exitStatus);
containerStatus.setState(ContainerState.COMPLETE);
return containerStatus;
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Thu Sep 19 23:42:10 2013
@@ -80,6 +80,8 @@ import org.apache.hadoop.yarn.server.uti
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
+
@LimitedPrivate("yarn")
@Evolving
@SuppressWarnings("unchecked")
@@ -179,13 +181,15 @@ public class CapacityScheduler
private Resource minimumAllocation;
private Resource maximumAllocation;
- private Map<ApplicationAttemptId, FiCaSchedulerApp> applications =
+ @VisibleForTesting
+ protected Map<ApplicationAttemptId, FiCaSchedulerApp> applications =
new ConcurrentHashMap<ApplicationAttemptId, FiCaSchedulerApp>();
private boolean initialized = false;
private ResourceCalculator calculator;
-
+ private boolean usePortForNodeName;
+
public CapacityScheduler() {}
@Override
@@ -256,6 +260,7 @@ public class CapacityScheduler
this.minimumAllocation = this.conf.getMinimumAllocation();
this.maximumAllocation = this.conf.getMaximumAllocation();
this.calculator = this.conf.getResourceCalculator();
+ this.usePortForNodeName = this.conf.getUsePortForNodeName();
this.rmContext = rmContext;
@@ -759,7 +764,8 @@ public class CapacityScheduler
}
private synchronized void addNode(RMNode nodeManager) {
- this.nodes.put(nodeManager.getNodeID(), new FiCaSchedulerNode(nodeManager));
+ this.nodes.put(nodeManager.getNodeID(), new FiCaSchedulerNode(nodeManager,
+ usePortForNodeName));
Resources.addTo(clusterResource, nodeManager.getTotalCapability());
root.updateClusterResource(clusterResource);
++numNodeManagers;
@@ -901,7 +907,7 @@ public class CapacityScheduler
LOG.debug("KILL_CONTAINER: container" + cont.toString());
}
completedContainer(cont,
- SchedulerUtils.createAbnormalContainerStatus(
+ SchedulerUtils.createPreemptedContainerStatus(
cont.getContainerId(),"Container being forcibly preempted:"
+ cont.getContainerId()),
RMContainerEventType.KILL);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Thu Sep 19 23:42:10 2013
@@ -338,6 +338,11 @@ public class CapacitySchedulerConfigurat
this);
}
+ public boolean getUsePortForNodeName() {
+ return getBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
+ }
+
public void setResourceComparator(
Class<? extends ResourceCalculator> resourceCalculatorClass) {
setClass(
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Thu Sep 19 23:42:10 2013
@@ -801,7 +801,7 @@ public class LeafQueue implements CSQueu
assignContainers(Resource clusterResource, FiCaSchedulerNode node) {
if(LOG.isDebugEnabled()) {
- LOG.debug("assignContainers: node=" + node.getHostName()
+ LOG.debug("assignContainers: node=" + node.getNodeName()
+ " #applications=" + activeApplications.size());
}
@@ -1130,7 +1130,7 @@ public class LeafQueue implements CSQueu
// Data-local
ResourceRequest nodeLocalResourceRequest =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeName());
if (nodeLocalResourceRequest != null) {
assigned =
assignNodeLocalContainers(clusterResource, nodeLocalResourceRequest,
@@ -1257,7 +1257,7 @@ public class LeafQueue implements CSQueu
if (type == NodeType.NODE_LOCAL) {
// Now check if we need containers on this host...
ResourceRequest nodeLocalRequest =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeName());
if (nodeLocalRequest != null) {
return nodeLocalRequest.getNumContainers() > 0;
}
@@ -1302,15 +1302,21 @@ public class LeafQueue implements CSQueu
FiCaSchedulerApp application, Priority priority,
ResourceRequest request, NodeType type, RMContainer rmContainer) {
if (LOG.isDebugEnabled()) {
- LOG.debug("assignContainers: node=" + node.getHostName()
+ LOG.debug("assignContainers: node=" + node.getNodeName()
+ " application=" + application.getApplicationId().getId()
+ " priority=" + priority.getPriority()
+ " request=" + request + " type=" + type);
}
Resource capability = request.getCapability();
-
Resource available = node.getAvailableResource();
+ Resource totalResource = node.getTotalResource();
+ if (!Resources.fitsIn(capability, totalResource)) {
+ LOG.warn("Node : " + node.getNodeID()
+ + " does not have sufficient resource for request : " + request
+ + " node total capability : " + node.getTotalResource());
+ return Resources.none();
+ }
assert Resources.greaterThan(
resourceCalculator, clusterResource, available, Resources.none());
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java Thu Sep 19 23:42:10 2013
@@ -49,6 +49,7 @@ public class FiCaSchedulerNode extends S
private Resource availableResource = recordFactory.newRecordInstance(Resource.class);
private Resource usedResource = recordFactory.newRecordInstance(Resource.class);
+ private Resource totalResourceCapability;
private volatile int numContainers;
@@ -59,11 +60,20 @@ public class FiCaSchedulerNode extends S
new HashMap<ContainerId, RMContainer>();
private final RMNode rmNode;
+ private final String nodeName;
- public FiCaSchedulerNode(RMNode node) {
+ public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName) {
this.rmNode = node;
this.availableResource.setMemory(node.getTotalCapability().getMemory());
this.availableResource.setVirtualCores(node.getTotalCapability().getVirtualCores());
+ totalResourceCapability =
+ Resource.newInstance(node.getTotalCapability().getMemory(), node
+ .getTotalCapability().getVirtualCores());
+ if (usePortForNodeName) {
+ nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort();
+ } else {
+ nodeName = rmNode.getHostName();
+ }
}
public RMNode getRMNode() {
@@ -79,8 +89,8 @@ public class FiCaSchedulerNode extends S
}
@Override
- public String getHostName() {
- return this.rmNode.getHostName();
+ public String getNodeName() {
+ return nodeName;
}
@Override
@@ -120,6 +130,11 @@ public class FiCaSchedulerNode extends S
return this.usedResource;
}
+ @Override
+ public Resource getTotalResource() {
+ return this.totalResourceCapability;
+ }
+
private synchronized boolean isValidContainer(Container c) {
if (launchedContainers.containsKey(c.getId()))
return true;
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java Thu Sep 19 23:42:10 2013
@@ -24,9 +24,9 @@ public class FiCaSchedulerUtils {
public static boolean isBlacklisted(FiCaSchedulerApp application,
FiCaSchedulerNode node, Log LOG) {
- if (application.isBlacklisted(node.getHostName())) {
+ if (application.isBlacklisted(node.getNodeName())) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Skipping 'host' " + node.getHostName() +
+ LOG.debug("Skipping 'host' " + node.getNodeName() +
" for " + application.getApplicationId() +
" since it has been blacklisted");
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Thu Sep 19 23:42:10 2013
@@ -185,7 +185,7 @@ public class AppSchedulable extends Sche
*/
private void reserve(Priority priority, FSSchedulerNode node,
Container container, boolean alreadyReserved) {
- LOG.info("Making reservation: node=" + node.getHostName() +
+ LOG.info("Making reservation: node=" + node.getNodeName() +
" app_id=" + app.getApplicationId());
if (!alreadyReserved) {
getMetrics().reserveResource(app.getUser(), container.getResource());
@@ -309,7 +309,7 @@ public class AppSchedulable extends Sche
ResourceRequest rackLocalRequest = app.getResourceRequest(priority,
node.getRackName());
ResourceRequest localRequest = app.getResourceRequest(priority,
- node.getHostName());
+ node.getNodeName());
if (localRequest != null && !localRequest.getRelaxLocality()) {
LOG.warn("Relax locality off is not supported on local request: "
@@ -369,7 +369,7 @@ public class AppSchedulable extends Sche
public boolean hasContainerForNode(Priority prio, FSSchedulerNode node) {
ResourceRequest anyRequest = app.getResourceRequest(prio, ResourceRequest.ANY);
ResourceRequest rackRequest = app.getResourceRequest(prio, node.getRackName());
- ResourceRequest nodeRequest = app.getResourceRequest(prio, node.getHostName());
+ ResourceRequest nodeRequest = app.getResourceRequest(prio, node.getNodeName());
return
// There must be outstanding requests at the given priority:
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java Thu Sep 19 23:42:10 2013
@@ -52,6 +52,7 @@ public class FSSchedulerNode extends Sch
private Resource availableResource;
private Resource usedResource = recordFactory.newRecordInstance(Resource.class);
+ private Resource totalResourceCapability;
private volatile int numContainers;
@@ -63,10 +64,19 @@ public class FSSchedulerNode extends Sch
new HashMap<ContainerId, RMContainer>();
private final RMNode rmNode;
+ private final String nodeName;
- public FSSchedulerNode(RMNode node) {
+ public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
this.rmNode = node;
this.availableResource = Resources.clone(node.getTotalCapability());
+ totalResourceCapability =
+ Resource.newInstance(node.getTotalCapability().getMemory(), node
+ .getTotalCapability().getVirtualCores());
+ if (usePortForNodeName) {
+ nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort();
+ } else {
+ nodeName = rmNode.getHostName();
+ }
}
public RMNode getRMNode() {
@@ -82,8 +92,8 @@ public class FSSchedulerNode extends Sch
}
@Override
- public String getHostName() {
- return rmNode.getHostName();
+ public String getNodeName() {
+ return nodeName;
}
@Override
@@ -167,6 +177,11 @@ public class FSSchedulerNode extends Sch
Resources.subtractFrom(usedResource, resource);
}
+ @Override
+ public Resource getTotalResource() {
+ return this.totalResourceCapability;
+ }
+
private synchronized void deductAvailableResource(Resource resource) {
if (resource == null) {
LOG.error("Invalid deduction of null resource for "
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Thu Sep 19 23:42:10 2013
@@ -35,10 +35,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -51,6 +49,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
@@ -122,6 +121,7 @@ public class FairScheduler implements Re
private Resource incrAllocation;
private QueueManager queueMgr;
private Clock clock;
+ private boolean usePortForNodeName;
private static final Log LOG = LogFactory.getLog(FairScheduler.class);
@@ -154,8 +154,9 @@ public class FairScheduler implements Re
// This stores per-application scheduling information, indexed by
// attempt ID's for fast lookup.
+ @VisibleForTesting
protected Map<ApplicationAttemptId, FSSchedulerApp> applications =
- new HashMap<ApplicationAttemptId, FSSchedulerApp>();
+ new ConcurrentHashMap<ApplicationAttemptId, FSSchedulerApp>();
// Nodes in the cluster, indexed by NodeId
private Map<NodeId, FSSchedulerNode> nodes =
@@ -385,8 +386,8 @@ public class FairScheduler implements Re
// Sort containers into reverse order of priority
Collections.sort(runningContainers, new Comparator<RMContainer>() {
public int compare(RMContainer c1, RMContainer c2) {
- int ret = c2.getContainer().getPriority().compareTo(
- c1.getContainer().getPriority());
+ int ret = c1.getContainer().getPriority().compareTo(
+ c2.getContainer().getPriority());
if (ret == 0) {
return c2.getContainerId().compareTo(c1.getContainerId());
}
@@ -444,7 +445,7 @@ public class FairScheduler implements Re
// proceed with kill
if (time + waitTimeBeforeKill < clock.getTime()) {
ContainerStatus status =
- SchedulerUtils.createAbnormalContainerStatus(
+ SchedulerUtils.createPreemptedContainerStatus(
container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER);
// TODO: Not sure if this ever actually adds this to the list of cleanup
@@ -751,7 +752,7 @@ public class FairScheduler implements Re
}
private synchronized void addNode(RMNode node) {
- nodes.put(node.getNodeID(), new FSSchedulerNode(node));
+ nodes.put(node.getNodeID(), new FSSchedulerNode(node, usePortForNodeName));
Resources.addTo(clusterCapacity, node.getTotalCapability());
updateRootQueueMetrics();
@@ -1065,7 +1066,8 @@ public class FairScheduler implements Re
sizeBasedWeight = this.conf.getSizeBasedWeight();
preemptionInterval = this.conf.getPreemptionInterval();
waitTimeBeforeKill = this.conf.getWaitTimeBeforeKill();
-
+ usePortForNodeName = this.conf.getUsePortForNodeName();
+
if (!initialized) {
rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf);
this.rmContext = rmContext;
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java Thu Sep 19 23:42:10 2013
@@ -166,7 +166,12 @@ public class FairSchedulerConfiguration
public int getWaitTimeBeforeKill() {
return getInt(WAIT_TIME_BEFORE_KILL, DEFAULT_WAIT_TIME_BEFORE_KILL);
}
-
+
+ public boolean getUsePortForNodeName() {
+ return getBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
+ }
+
/**
* Parses a resource config value of a form like "1024", "1024 mb",
* or "1024 mb, 3 vcores". If no units are given, megabytes are assumed.
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Thu Sep 19 23:42:10 2013
@@ -25,8 +25,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,12 +36,10 @@ import org.apache.hadoop.conf.Configurab
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -50,7 +48,9 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
@@ -90,6 +90,8 @@ import org.apache.hadoop.yarn.util.resou
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
+
@LimitedPrivate("yarn")
@Evolving
@SuppressWarnings("unchecked")
@@ -111,9 +113,12 @@ public class FifoScheduler implements Re
private boolean initialized;
private Resource minimumAllocation;
private Resource maximumAllocation;
+ private boolean usePortForNodeName;
- private Map<ApplicationAttemptId, FiCaSchedulerApp> applications
- = new TreeMap<ApplicationAttemptId, FiCaSchedulerApp>();
+ // Use ConcurrentSkipListMap because applications need to be ordered
+ @VisibleForTesting
+ protected Map<ApplicationAttemptId, FiCaSchedulerApp> applications
+ = new ConcurrentSkipListMap<ApplicationAttemptId, FiCaSchedulerApp>();
private ActiveUsersManager activeUsersManager;
@@ -233,6 +238,9 @@ public class FifoScheduler implements Re
Resources.createResource(conf.getInt(
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
+ this.usePortForNodeName = conf.getBoolean(
+ YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
conf);
this.activeUsersManager = new ActiveUsersManager(metrics);
@@ -490,7 +498,7 @@ public class FifoScheduler implements Re
FiCaSchedulerApp application, Priority priority) {
int assignedContainers = 0;
ResourceRequest request =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeName());
if (request != null) {
// Don't allocate on this node if we don't need containers on this rack
ResourceRequest rackRequest =
@@ -801,7 +809,8 @@ public class FifoScheduler implements Re
}
private synchronized void addNode(RMNode nodeManager) {
- this.nodes.put(nodeManager.getNodeID(), new FiCaSchedulerNode(nodeManager));
+ this.nodes.put(nodeManager.getNodeID(), new FiCaSchedulerNode(nodeManager,
+ usePortForNodeName));
Resources.addTo(clusterResource, nodeManager.getTotalCapability());
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java Thu Sep 19 23:42:10 2013
@@ -33,9 +33,18 @@ public class ClientToAMTokenSecretManage
private Map<ApplicationAttemptId, SecretKey> masterKeys =
new HashMap<ApplicationAttemptId, SecretKey>();
- public synchronized void registerApplication(
+ public synchronized SecretKey registerApplication(
ApplicationAttemptId applicationAttemptID) {
- this.masterKeys.put(applicationAttemptID, generateSecret());
+ SecretKey key = generateSecret();
+ this.masterKeys.put(applicationAttemptID, key);
+ return key;
+ }
+
+ public synchronized SecretKey registerMasterKey(
+ ApplicationAttemptId applicationAttemptID, byte[] keyData) {
+ SecretKey key = createSecretKey(keyData);
+ this.masterKeys.put(applicationAttemptID, key);
+ return key;
}
public synchronized void unRegisterApplication(
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java Thu Sep 19 23:42:10 2013
@@ -34,6 +34,7 @@ import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -47,6 +48,8 @@ import org.apache.hadoop.service.Abstrac
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import com.google.common.annotations.VisibleForTesting;
@@ -64,6 +67,7 @@ public class DelegationTokenRenewer exte
// global single timer (daemon)
private Timer renewalTimer;
+ private RMContext rmContext;
// delegation token canceler thread
private DelegationTokenCancelThread dtCancelThread =
@@ -80,6 +84,9 @@ public class DelegationTokenRenewer exte
private long tokenRemovalDelayMs;
private Thread delayedRemovalThread;
+ private boolean isServiceStarted = false;
+ private List<DelegationTokenToRenew> pendingTokenForRenewal =
+ new ArrayList<DelegationTokenRenewer.DelegationTokenToRenew>();
private boolean tokenKeepAliveEnabled;
@@ -100,7 +107,6 @@ public class DelegationTokenRenewer exte
@Override
protected void serviceStart() throws Exception {
-
dtCancelThread.start();
renewalTimer = new Timer(true);
if (tokenKeepAliveEnabled) {
@@ -109,6 +115,15 @@ public class DelegationTokenRenewer exte
"DelayedTokenCanceller");
delayedRemovalThread.start();
}
+ // enable RM to short-circuit token operations directly to itself
+ RMDelegationTokenIdentifier.Renewer.setSecretManager(
+ rmContext.getRMDelegationTokenSecretManager(),
+ rmContext.getClientRMService().getBindAddress());
+ // Delegation token renewal is delayed until ClientRMService starts. As
+ // it is required to short circuit the token renewal calls.
+ isServiceStarted = true;
+ renewIfServiceIsStarted(pendingTokenForRenewal);
+ pendingTokenForRenewal.clear();
super.serviceStart();
}
@@ -275,8 +290,8 @@ public class DelegationTokenRenewer exte
* @throws IOException
*/
public void addApplication(
- ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd)
- throws IOException {
+ ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd)
+ throws IOException {
if (ts == null) {
return; //nothing to add
}
@@ -291,25 +306,40 @@ public class DelegationTokenRenewer exte
// find tokens for renewal, but don't add timers until we know
// all renewable tokens are valid
- Set<DelegationTokenToRenew> dtrs = new HashSet<DelegationTokenToRenew>();
+ // At RM restart it is safe to assume that all the previously added tokens
+ // are valid
+ List<DelegationTokenToRenew> tokenList =
+ new ArrayList<DelegationTokenRenewer.DelegationTokenToRenew>();
for(Token<?> token : tokens) {
- // first renew happens immediately
if (token.isManaged()) {
- DelegationTokenToRenew dtr =
- new DelegationTokenToRenew(applicationId, token, getConfig(), now,
- shouldCancelAtEnd);
- renewToken(dtr);
- dtrs.add(dtr);
+ tokenList.add(new DelegationTokenToRenew(applicationId,
+ token, getConfig(), now, shouldCancelAtEnd));
}
}
- for (DelegationTokenToRenew dtr : dtrs) {
- addTokenToList(dtr);
- setTimerForTokenRenewal(dtr);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Registering token for renewal for:" +
- " service = " + dtr.token.getService() +
- " for appId = " + applicationId);
+ if (!tokenList.isEmpty()){
+ renewIfServiceIsStarted(tokenList);
+ }
+ }
+
+ protected void renewIfServiceIsStarted(List<DelegationTokenToRenew> dtrs)
+ throws IOException {
+ if (isServiceStarted) {
+ // Renewing token and adding it to timer calls are separated purposefully
+ // If user provides incorrect token then it should not be added for
+ // renewal.
+ for (DelegationTokenToRenew dtr : dtrs) {
+ renewToken(dtr);
+ }
+ for (DelegationTokenToRenew dtr : dtrs) {
+ addTokenToList(dtr);
+ setTimerForTokenRenewal(dtr);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Registering token for renewal for:" + " service = "
+ + dtr.token.getService() + " for appId = " + dtr.applicationId);
+ }
}
+ } else {
+ pendingTokenForRenewal.addAll(dtrs);
}
}
@@ -513,4 +543,7 @@ public class DelegationTokenRenewer exte
}
}
+ public void setRMContext(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Thu Sep 19 23:42:10 2013
@@ -155,10 +155,10 @@ class NodesPage extends RmView {
private String nodesTableInit() {
StringBuilder b = tableInit().append(", aoColumnDefs: [");
- b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
+ b.append("{'bSearchable': false, 'aTargets': [ 6 ]}");
b.append(", {'sType': 'title-numeric', 'bSearchable': false, " +
- "'aTargets': [ 8, 9 ] }");
- b.append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}");
+ "'aTargets': [ 7, 8 ] }");
+ b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}");
b.append("]}");
return b.toString();
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Thu Sep 19 23:42:10 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
@@ -231,6 +232,7 @@ public class RMWebServices {
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppsInfo getApps(@Context HttpServletRequest hsr,
@QueryParam("state") String stateQuery,
+ @QueryParam("states") Set<String> statesQuery,
@QueryParam("finalStatus") String finalStatusQuery,
@QueryParam("user") String userQuery,
@QueryParam("queue") String queueQuery,
@@ -245,6 +247,7 @@ public class RMWebServices {
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
+ boolean checkAppStates = false;
long countNum = 0;
// set values suitable in case both of begin/end not specified
@@ -321,6 +324,36 @@ public class RMWebServices {
checkAppTypes = true;
}
+ String allAppStates;
+ RMAppState[] stateArray = RMAppState.values();
+ allAppStates = Arrays.toString(stateArray);
+
+ Set<String> appStates = new HashSet<String>();
+ // stateQuery is deprecated.
+ if (stateQuery != null && !stateQuery.isEmpty()) {
+ statesQuery.add(stateQuery);
+ }
+ if (!statesQuery.isEmpty()) {
+ for (String applicationState : statesQuery) {
+ if (applicationState != null && !applicationState.isEmpty()) {
+ String[] states = applicationState.split(",");
+ for (String state : states) {
+ try {
+ RMAppState.valueOf(state.trim());
+ } catch (IllegalArgumentException iae) {
+ throw new BadRequestException(
+ "Invalid application-state " + state
+ + " specified. It should be one of " + allAppStates);
+ }
+ appStates.add(state.trim().toLowerCase());
+ }
+ }
+ }
+ }
+ if (!appStates.isEmpty()) {
+ checkAppStates = true;
+ }
+
final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext()
.getRMApps();
AppsInfo allApps = new AppsInfo();
@@ -329,11 +362,10 @@ public class RMWebServices {
if (checkCount && num == countNum) {
break;
}
- if (stateQuery != null && !stateQuery.isEmpty()) {
- RMAppState.valueOf(stateQuery);
- if (!rmapp.getState().toString().equalsIgnoreCase(stateQuery)) {
- continue;
- }
+
+ if (checkAppStates
+ && !appStates.contains(rmapp.getState().toString().toLowerCase())) {
+ continue;
}
if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
FinalApplicationStatus.valueOf(finalStatusQuery);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java?rev=1524865&r1=1524864&r2=1524865&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java Thu Sep 19 23:42:10 2013
@@ -84,7 +84,7 @@ public class ClusterMetricsInfo {
this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
this.totalNodes = activeNodes + lostNodes + decommissionedNodes
- + rebootedNodes;
+ + rebootedNodes + unhealthyNodes;
}
public int getAppsSubmitted() {