You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2013/12/02 18:41:53 UTC
svn commit: r1547122 [2/2] - in
/hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/proto/server/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/...
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java Mon Dec 2 17:41:44 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.security.Groups;
@@ -58,7 +59,7 @@ public abstract class QueuePlacementRule
*/
public String assignAppToQueue(String requestedQueue, String user,
Groups groups, Collection<String> configuredQueues) throws IOException {
- String queue = getQueueForApp(requestedQueue, user, groups);
+ String queue = getQueueForApp(requestedQueue, user, groups, configuredQueues);
if (create || configuredQueues.contains(queue)) {
return queue;
} else {
@@ -103,7 +104,7 @@ public abstract class QueuePlacementRule
* continue to the next rule.
*/
protected abstract String getQueueForApp(String requestedQueue, String user,
- Groups groups) throws IOException;
+ Groups groups, Collection<String> configuredQueues) throws IOException;
/**
* Places apps in queues by username of the submitter
@@ -111,7 +112,7 @@ public abstract class QueuePlacementRule
public static class User extends QueuePlacementRule {
@Override
protected String getQueueForApp(String requestedQueue,
- String user, Groups groups) {
+ String user, Groups groups, Collection<String> configuredQueues) {
return "root." + user;
}
@@ -127,7 +128,8 @@ public abstract class QueuePlacementRule
public static class PrimaryGroup extends QueuePlacementRule {
@Override
protected String getQueueForApp(String requestedQueue,
- String user, Groups groups) throws IOException {
+ String user, Groups groups,
+ Collection<String> configuredQueues) throws IOException {
return "root." + groups.getGroups(user).get(0);
}
@@ -136,6 +138,33 @@ public abstract class QueuePlacementRule
return create;
}
}
+
+ /**
+ * Places apps in queues by secondary group of the submitter
+ *
+ * Match will be made on first secondary group that exist in
+ * queues
+ */
+ public static class SecondaryGroupExistingQueue extends QueuePlacementRule {
+ @Override
+ protected String getQueueForApp(String requestedQueue,
+ String user, Groups groups,
+ Collection<String> configuredQueues) throws IOException {
+ List<String> groupNames = groups.getGroups(user);
+ for (int i = 1; i < groupNames.size(); i++) {
+ if (configuredQueues.contains("root." + groupNames.get(i))) {
+ return "root." + groupNames.get(i);
+ }
+ }
+
+ return "";
+ }
+
+ @Override
+ public boolean isTerminal() {
+ return create;
+ }
+ }
/**
* Places apps in queues by requested queue of the submitter
@@ -143,7 +172,7 @@ public abstract class QueuePlacementRule
public static class Specified extends QueuePlacementRule {
@Override
protected String getQueueForApp(String requestedQueue,
- String user, Groups groups) {
+ String user, Groups groups, Collection<String> configuredQueues) {
if (requestedQueue.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)) {
return "";
} else {
@@ -166,7 +195,7 @@ public abstract class QueuePlacementRule
public static class Default extends QueuePlacementRule {
@Override
protected String getQueueForApp(String requestedQueue, String user,
- Groups groups) {
+ Groups groups, Collection<String> configuredQueues) {
return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
}
@@ -188,7 +217,7 @@ public abstract class QueuePlacementRule
@Override
protected String getQueueForApp(String requestedQueue, String user,
- Groups groups) {
+ Groups groups, Collection<String> configuredQueues) {
throw new UnsupportedOperationException();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java Mon Dec 2 17:41:44 2013
@@ -39,7 +39,7 @@ public class FairSchedulerLeafQueueInfo
public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
super(queue, scheduler);
- Collection<AppSchedulable> apps = queue.getAppSchedulables();
+ Collection<AppSchedulable> apps = queue.getRunnableAppSchedulables();
for (AppSchedulable app : apps) {
if (app.getApp().isPending()) {
numPendingApps++;
@@ -47,6 +47,7 @@ public class FairSchedulerLeafQueueInfo
numActiveApps++;
}
}
+ numPendingApps += queue.getNonRunnableAppSchedulables().size();
}
public int getNumActiveApplications() {
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java Mon Dec 2 17:41:44 2013
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMDTSecretManagerState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -106,6 +107,8 @@ public class RMStateStoreTestBase extend
interface RMStateStoreHelper {
RMStateStore getRMStateStore() throws Exception;
boolean isFinalStateValid() throws Exception;
+ void writeVersion(RMStateVersion version) throws Exception;
+ RMStateVersion getCurrentVersion() throws Exception;
}
void waitNotify(TestDispatcher dispatcher) {
@@ -379,4 +382,37 @@ public class RMStateStoreTestBase extend
appToken.setService(new Text("appToken service"));
return appToken;
}
+
+ public void testCheckVersion(RMStateStoreHelper stateStoreHelper)
+ throws Exception {
+ RMStateStore store = stateStoreHelper.getRMStateStore();
+ store.setRMDispatcher(new TestDispatcher());
+
+ // default version
+ RMStateVersion defaultVersion = stateStoreHelper.getCurrentVersion();
+ store.checkVersion();
+ Assert.assertEquals(defaultVersion, store.loadVersion());
+
+ // compatible version
+ RMStateVersion compatibleVersion =
+ RMStateVersion.newInstance(defaultVersion.getMajorVersion(),
+ defaultVersion.getMinorVersion() + 2);
+ stateStoreHelper.writeVersion(compatibleVersion);
+ Assert.assertEquals(compatibleVersion, store.loadVersion());
+ store.checkVersion();
+ // overwrite the compatible version
+ Assert.assertEquals(defaultVersion, store.loadVersion());
+
+ // incompatible version
+ RMStateVersion incompatibleVersion =
+ RMStateVersion.newInstance(defaultVersion.getMajorVersion() + 2,
+ defaultVersion.getMinorVersion());
+ stateStoreHelper.writeVersion(incompatibleVersion);
+ try {
+ store.checkVersion();
+ Assert.fail("Invalid version, should fail.");
+ } catch (Throwable t) {
+ Assert.assertTrue(t instanceof RMStateVersionIncompatibleException);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java Mon Dec 2 17:41:44 2013
@@ -32,6 +32,8 @@ import org.apache.hadoop.hdfs.HdfsConfig
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
@@ -42,7 +44,7 @@ public class TestFSRMStateStore extends
class TestFSRMStateStoreTester implements RMStateStoreHelper {
Path workingDirPathURI;
- FileSystemRMStateStore store;
+ TestFileSystemRMStore store;
MiniDFSCluster cluster;
class TestFileSystemRMStore extends FileSystemRMStateStore {
@@ -54,6 +56,14 @@ public class TestFSRMStateStore extends
start();
Assert.assertNotNull(fs);
}
+
+ public Path getVersionNode() {
+ return new Path(new Path(workingDirPathURI, ROOT_DIR_NAME), VERSION_NODE);
+ }
+
+ public RMStateVersion getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
}
public TestFSRMStateStoreTester(MiniDFSCluster cluster) throws Exception {
@@ -81,6 +91,17 @@ public class TestFSRMStateStore extends
FileStatus[] files = fs.listStatus(workingDirPathURI);
return files.length == 1;
}
+
+ @Override
+ public void writeVersion(RMStateVersion version) throws Exception {
+ store.updateFile(store.getVersionNode(), ((RMStateVersionPBImpl) version)
+ .getProto().toByteArray());
+ }
+
+ @Override
+ public RMStateVersion getCurrentVersion() throws Exception {
+ return store.getCurrentVersion();
+ }
}
@Test
@@ -113,6 +134,7 @@ public class TestFSRMStateStore extends
Assert.assertFalse(fileSystemRMStateStore.fsWorkingPath
.getFileSystem(conf).exists(tempAppAttemptFile));
testRMDTSecretManagerStateStore(fsTester);
+ testCheckVersion(fsTester);
} finally {
cluster.shutdown();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Mon Dec 2 17:41:44 2013
@@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.conf.HAUti
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl;
import org.apache.zookeeper.ZooKeeper;
import org.junit.Test;
@@ -54,7 +56,7 @@ public class TestZKRMStateStore extends
class TestZKRMStateStoreTester implements RMStateStoreHelper {
ZooKeeper client;
- ZKRMStateStore store;
+ TestZKRMStateStoreInternal store;
class TestZKRMStateStoreInternal extends ZKRMStateStore {
@@ -69,6 +71,14 @@ public class TestZKRMStateStore extends
public ZooKeeper getNewZooKeeper() throws IOException {
return client;
}
+
+ public String getVersionNode() {
+ return znodeWorkingPath + "/" + ROOT_ZNODE_NAME + "/" + VERSION_NODE;
+ }
+
+ public RMStateVersion getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
}
public RMStateStore getRMStateStore() throws Exception {
@@ -86,6 +96,17 @@ public class TestZKRMStateStore extends
List<String> nodes = client.getChildren(store.znodeWorkingPath, false);
return nodes.size() == 1;
}
+
+ @Override
+ public void writeVersion(RMStateVersion version) throws Exception {
+ client.setData(store.getVersionNode(), ((RMStateVersionPBImpl) version)
+ .getProto().toByteArray(), -1);
+ }
+
+ @Override
+ public RMStateVersion getCurrentVersion() throws Exception {
+ return store.getCurrentVersion();
+ }
}
@Test
@@ -93,6 +114,7 @@ public class TestZKRMStateStore extends
TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
testRMAppStateStore(zkTester);
testRMDTSecretManagerStateStore(zkTester);
+ testCheckVersion(zkTester);
}
private Configuration createHARMConf(
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Mon Dec 2 17:41:44 2013
@@ -18,19 +18,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assume.assumeTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
import junit.framework.Assert;
@@ -41,13 +36,10 @@ import org.apache.hadoop.security.Securi
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -64,11 +56,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
@@ -76,7 +65,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -290,7 +278,6 @@ public class TestRMAppTransitions {
// test to make sure times are set when app finishes
private void assertTimesAtFinish(RMApp application) {
- sendAppUpdateSavedEvent(application);
assertStartTimeSet(application);
Assert.assertTrue("application finish time is not greater then 0",
(application.getFinishTime() > 0));
@@ -303,7 +290,6 @@ public class TestRMAppTransitions {
}
private void assertKilled(RMApp application) {
- sendAppUpdateSavedEvent(application);
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
assertFinalAppStatus(FinalApplicationStatus.KILLED, application);
@@ -314,6 +300,7 @@ public class TestRMAppTransitions {
private void assertAppAndAttemptKilled(RMApp application)
throws InterruptedException {
+ sendAppUpdateSavedEvent(application);
assertKilled(application);
// send attempt final state saved event.
application.getCurrentAppAttempt().handle(
@@ -325,7 +312,6 @@ public class TestRMAppTransitions {
}
private void assertFailed(RMApp application, String regex) {
- sendAppUpdateSavedEvent(application);
assertTimesAtFinish(application);
assertAppState(RMAppState.FAILED, application);
assertFinalAppStatus(FinalApplicationStatus.FAILED, application);
@@ -475,6 +461,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
RMAppAttempt appAttempt = application.getCurrentAppAttempt();
Assert.assertEquals(1, appAttempt.getAppAttemptId().getAttemptId());
+ sendAppUpdateSavedEvent(application);
assertFailed(application,
".*Unmanaged application.*Failing the application.*");
}
@@ -504,6 +491,7 @@ public class TestRMAppTransitions {
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL);
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertKilled(application);
}
@@ -518,6 +506,7 @@ public class TestRMAppTransitions {
new RMAppRejectedEvent(application.getApplicationId(), rejectedText);
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
}
@@ -531,6 +520,7 @@ public class TestRMAppTransitions {
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL);
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertKilled(application);
}
@@ -545,6 +535,7 @@ public class TestRMAppTransitions {
new RMAppRejectedEvent(application.getApplicationId(), rejectedText);
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
}
@@ -559,6 +550,7 @@ public class TestRMAppTransitions {
new RMAppRejectedEvent(application.getApplicationId(), rejectedText);
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
}
@@ -603,6 +595,7 @@ public class TestRMAppTransitions {
RMAppEventType.ATTEMPT_FAILED, message);
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertFailed(application, ".*" + message + ".*Failing the application.*");
}
@@ -635,7 +628,7 @@ public class TestRMAppTransitions {
new RMAppFinishedAttemptEvent(application.getApplicationId(), null);
application.handle(finishEvent);
assertAppState(RMAppState.FINAL_SAVING, application);
-
+ sendAppUpdateSavedEvent(application);
assertKilled(application);
}
@@ -681,6 +674,7 @@ public class TestRMAppTransitions {
RMAppEventType.ATTEMPT_FAILED, "");
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertFailed(application, ".*Failing the application.*");
// FAILED => FAILED event RMAppEventType.KILL
@@ -758,6 +752,7 @@ public class TestRMAppTransitions {
new RMAppRejectedEvent(application.getApplicationId(), "");
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertTimesAtFinish(application);
assertAppState(RMAppState.FAILED, application);
@@ -769,10 +764,6 @@ public class TestRMAppTransitions {
assertTimesAtFinish(application);
assertAppState(RMAppState.FAILED, application);
- // FAILED => FAILED event RMAppEventType.APP_SAVED
- event = new RMAppNewSavedEvent(application.getApplicationId(), null);
- application.handle(event);
- rmDispatcher.await();
assertTimesAtFinish(application);
assertAppState(RMAppState.FAILED, application);
}
@@ -788,6 +779,7 @@ public class TestRMAppTransitions {
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL);
application.handle(event);
rmDispatcher.await();
+ sendAppUpdateSavedEvent(application);
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
@@ -824,10 +816,6 @@ public class TestRMAppTransitions {
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
- // KILLED => KILLED event RMAppEventType.APP_SAVED
- event = new RMAppNewSavedEvent(application.getApplicationId(), null);
- application.handle(event);
- rmDispatcher.await();
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
}
@@ -841,70 +829,4 @@ public class TestRMAppTransitions {
report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
}
-
- @Test
- public void testClientTokens() throws Exception {
- assumeTrue(isSecurityEnabled);
-
- RMApp app = createNewTestApp(null);
- assertAppState(RMAppState.NEW, app);
- ApplicationReport report = app.createAndGetApplicationReport(null, true);
- Assert.assertNull(report.getClientToAMToken());
- report = app.createAndGetApplicationReport("clientuser", true);
- Assert.assertNull(report.getClientToAMToken());
-
- app = testCreateAppRunning(null);
- rmDispatcher.await();
- assertAppState(RMAppState.RUNNING, app);
-
- report = app.createAndGetApplicationReport("clientuser", true);
- Assert.assertNull(report.getClientToAMToken());
-
- // this method is to make AMLaunchedTransition invoked inside which
- // ClientTokenMasterKey is registered in ClientTokenSecretManager
- moveCurrentAttemptToLaunchedState(app.getCurrentAppAttempt());
-
- report = app.createAndGetApplicationReport(null, true);
- Assert.assertNull(report.getClientToAMToken());
- report = app.createAndGetApplicationReport("clientuser", true);
- Assert.assertNotNull(report.getClientToAMToken());
-
- // kill the app attempt and verify client token is unavailable
- app.handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL));
- rmDispatcher.await();
- assertAppAndAttemptKilled(app);
- report = app.createAndGetApplicationReport(null, true);
- Assert.assertNull(report.getClientToAMToken());
- report = app.createAndGetApplicationReport("clientuser", true);
- Assert.assertNull(report.getClientToAMToken());
- }
-
- @SuppressWarnings("unchecked")
- private void moveCurrentAttemptToLaunchedState(RMAppAttempt attempt) {
- attempt.handle(new RMAppAttemptEvent(attempt.getAppAttemptId(),
- RMAppAttemptEventType.APP_ACCEPTED));
- // Mock the allocation of AM container
- Container container = mock(Container.class);
- Resource resource = BuilderUtils.newResource(2048, 1);
- when(container.getId()).thenReturn(
- BuilderUtils.newContainerId(attempt.getAppAttemptId(), 1));
- when(container.getResource()).thenReturn(resource);
- Allocation allocation = mock(Allocation.class);
- when(allocation.getContainers()).thenReturn(
- Collections.singletonList(container));
- when(allocation.getContainers()).
- thenReturn(Collections.singletonList(container));
- when(
- scheduler.allocate(any(ApplicationAttemptId.class), any(List.class),
- any(List.class), any(List.class), any(List.class))).thenReturn(
- allocation);
- attempt.handle(new RMAppAttemptContainerAllocatedEvent(attempt
- .getAppAttemptId(), container));
- attempt
- .handle(new RMAppAttemptNewSavedEvent(attempt.getAppAttemptId(), null));
- attempt.handle(new RMAppAttemptEvent(attempt.getAppAttemptId(),
- RMAppAttemptEventType.LAUNCHED));
-
- assertEquals(RMAppAttemptState.LAUNCHED, attempt.getAppAttemptState());
- }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Mon Dec 2 17:41:44 2013
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -44,6 +45,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -57,6 +59,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.event.InlineDispatcher;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
@@ -74,9 +77,9 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRejectedEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
@@ -1068,6 +1071,38 @@ public class TestRMAppAttemptTransitions
diagnostics, 0, false);
}
+ // this is to test user can get client tokens only after the client token
+ // master key is saved in the state store and also registered in
+ // ClientTokenSecretManager
+ @Test
+ public void testGetClientToken() throws Exception {
+ assumeTrue(isSecurityEnabled);
+ Container amContainer = allocateApplicationAttempt();
+
+ // before attempt is launched, can not get ClientToken
+ Token<ClientToAMTokenIdentifier> token =
+ applicationAttempt.createClientToken(null);
+ Assert.assertNull(token);
+ token = applicationAttempt.createClientToken("clientuser");
+ Assert.assertNull(token);
+
+ launchApplicationAttempt(amContainer);
+ // after attempt is launched , can get ClientToken
+ token = applicationAttempt.createClientToken(null);
+ Assert.assertNull(token);
+ token = applicationAttempt.createClientToken("clientuser");
+ Assert.assertNotNull(token);
+
+ applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
+ .getAppAttemptId(), RMAppAttemptEventType.KILL));
+ sendAttemptUpdateSavedEvent(applicationAttempt);
+ // after attempt is killed, can not get Client Token
+ token = applicationAttempt.createClientToken(null);
+ Assert.assertNull(token);
+ token = applicationAttempt.createClientToken("clientuser");
+ Assert.assertNull(token);
+ }
+
private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) {
verify(amRMTokenManager, times(count)).applicationMasterFinished(appAttemptId);
if (UserGroupInformation.isSecurityEnabled()) {
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Mon Dec 2 17:41:44 2013
@@ -596,23 +596,24 @@ public class TestCapacityScheduler {
public void testConcurrentAccessOnApplications() throws Exception {
CapacityScheduler cs = new CapacityScheduler();
verifyConcurrentAccessOnApplications(
- cs.applications, FiCaSchedulerApp.class);
+ cs.applications, FiCaSchedulerApp.class, Queue.class);
}
- public static <T extends SchedulerApplication>
+ public static <T extends SchedulerApplication, Q extends Queue>
void verifyConcurrentAccessOnApplications(
- final Map<ApplicationAttemptId, T> applications, Class<T> clazz)
+ final Map<ApplicationAttemptId, T> applications, Class<T> appClazz,
+ final Class<Q> queueClazz)
throws Exception {
final int size = 10000;
final ApplicationId appId = ApplicationId.newInstance(0, 0);
- final Constructor<T> ctor = clazz.getDeclaredConstructor(
- ApplicationAttemptId.class, String.class, Queue.class,
+ final Constructor<T> ctor = appClazz.getDeclaredConstructor(
+ ApplicationAttemptId.class, String.class, queueClazz,
ActiveUsersManager.class, RMContext.class);
ApplicationAttemptId appAttemptId0
= ApplicationAttemptId.newInstance(appId, 0);
applications.put(appAttemptId0, ctor.newInstance(
- appAttemptId0, null, mock(Queue.class), null, null));
+ appAttemptId0, null, mock(queueClazz), null, null));
assertNotNull(applications.get(appAttemptId0));
// Imitating the thread of scheduler that will add and remove apps
@@ -627,7 +628,7 @@ public class TestCapacityScheduler {
= ApplicationAttemptId.newInstance(appId, i);
try {
applications.put(appAttemptId, ctor.newInstance(
- appAttemptId, null, mock(Queue.class), null, null));
+ appAttemptId, null, mock(queueClazz), null, null));
} catch (Exception e) {
failed.set(true);
finished.set(true);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SimpleGroupsMapping.java Mon Dec 2 17:41:44 2013
@@ -28,7 +28,7 @@ public class SimpleGroupsMapping impleme
@Override
public List<String> getGroups(String user) {
- return Arrays.asList(user + "group");
+ return Arrays.asList(user + "group", user + "subgroup1", user + "subgroup2");
}
@Override
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java Mon Dec 2 17:41:44 2013
@@ -24,7 +24,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.util.Clock;
import org.junit.Test;
import org.mockito.Mockito;
@@ -53,7 +52,7 @@ public class TestFSSchedulerApp {
@Test
public void testDelayScheduling() {
- Queue queue = Mockito.mock(Queue.class);
+ FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
Priority prio = Mockito.mock(Priority.class);
Mockito.when(prio.getPriority()).thenReturn(1);
double nodeLocalityThreshold = .5;
@@ -110,7 +109,7 @@ public class TestFSSchedulerApp {
@Test
public void testDelaySchedulingForContinuousScheduling()
throws InterruptedException {
- Queue queue = Mockito.mock(Queue.class);
+ FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
Priority prio = Mockito.mock(Priority.class);
Mockito.when(prio.getPriority()).thenReturn(1);
@@ -170,7 +169,7 @@ public class TestFSSchedulerApp {
* no tin use), the least restrictive locality level is returned.
*/
public void testLocalityLevelWithoutDelays() {
- Queue queue = Mockito.mock(Queue.class);
+ FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
Priority prio = Mockito.mock(Priority.class);
Mockito.when(prio.getPriority()).thenReturn(1);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Mon Dec 2 17:41:44 2013
@@ -100,7 +100,7 @@ import com.google.common.collect.Sets;
public class TestFairScheduler {
- private class MockClock implements Clock {
+ static class MockClock implements Clock {
private long time = 0;
@Override
public long getTime() {
@@ -613,9 +613,9 @@ public class TestFairScheduler {
appAttemptId, "default", "user1");
scheduler.handle(appAddedEvent);
assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
- .getAppSchedulables().size());
+ .getRunnableAppSchedulables().size());
assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true)
- .getAppSchedulables().size());
+ .getRunnableAppSchedulables().size());
assertEquals("root.user1", rmApp.getQueue());
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
@@ -625,11 +625,11 @@ public class TestFairScheduler {
createAppAttemptId(2, 1), "default", "user2");
scheduler.handle(appAddedEvent2);
assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
- .getAppSchedulables().size());
+ .getRunnableAppSchedulables().size());
assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true)
- .getAppSchedulables().size());
+ .getRunnableAppSchedulables().size());
assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true)
- .getAppSchedulables().size());
+ .getRunnableAppSchedulables().size());
}
@Test
@@ -682,8 +682,10 @@ public class TestFairScheduler {
rules.add(new QueuePlacementRule.Specified().initialize(true, null));
rules.add(new QueuePlacementRule.User().initialize(false, null));
rules.add(new QueuePlacementRule.PrimaryGroup().initialize(false, null));
+ rules.add(new QueuePlacementRule.SecondaryGroupExistingQueue().initialize(false, null));
rules.add(new QueuePlacementRule.Default().initialize(true, null));
- Set<String> queues = Sets.newHashSet("root.user1", "root.user3group");
+ Set<String> queues = Sets.newHashSet("root.user1", "root.user3group",
+ "root.user4subgroup1", "root.user4subgroup2" , "root.user5subgroup2");
scheduler.getQueueManager().placementPolicy = new QueuePlacementPolicy(
rules, queues, conf);
appId = createSchedulingRequest(1024, "somequeue", "user1");
@@ -692,6 +694,10 @@ public class TestFairScheduler {
assertEquals("root.user1", apps.get(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "user3");
assertEquals("root.user3group", apps.get(appId).getQueueName());
+ appId = createSchedulingRequest(1024, "default", "user4");
+ assertEquals("root.user4subgroup1", apps.get(appId).getQueueName());
+ appId = createSchedulingRequest(1024, "default", "user5");
+ assertEquals("root.user5subgroup2", apps.get(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "otheruser");
assertEquals("root.default", apps.get(appId).getQueueName());
@@ -815,7 +821,7 @@ public class TestFairScheduler {
// That queue should have one app
assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
- .getAppSchedulables().size());
+ .getRunnableAppSchedulables().size());
AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
createAppAttemptId(1, 1), RMAppAttemptState.FINISHED);
@@ -825,7 +831,7 @@ public class TestFairScheduler {
// Queue should have no apps
assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
- .getAppSchedulables().size());
+ .getRunnableAppSchedulables().size());
}
@Test
@@ -2394,7 +2400,158 @@ public class TestFairScheduler {
public void testConcurrentAccessOnApplications() throws Exception {
FairScheduler fs = new FairScheduler();
TestCapacityScheduler.verifyConcurrentAccessOnApplications(
- fs.applications, FSSchedulerApp.class);
+ fs.applications, FSSchedulerApp.class, FSLeafQueue.class);
+ }
+
+
+ private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) {
+ FSSchedulerApp app = scheduler.applications.get(attId);
+ FSLeafQueue queue = app.getQueue();
+ Collection<AppSchedulable> runnableApps =
+ queue.getRunnableAppSchedulables();
+ Collection<AppSchedulable> nonRunnableApps =
+ queue.getNonRunnableAppSchedulables();
+ assertEquals(runnable, runnableApps.contains(app.getAppSchedulable()));
+ assertEquals(!runnable, nonRunnableApps.contains(app.getAppSchedulable()));
+ }
+
+ private void verifyQueueNumRunnable(String queueName, int numRunnableInQueue,
+ int numNonRunnableInQueue) {
+ FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue(queueName, false);
+ assertEquals(numRunnableInQueue,
+ queue.getRunnableAppSchedulables().size());
+ assertEquals(numNonRunnableInQueue,
+ queue.getNonRunnableAppSchedulables().size());
+ }
+
+ @Test
+ public void testUserAndQueueMaxRunningApps() throws Exception {
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queue1\">");
+ out.println("<maxRunningApps>2</maxRunningApps>");
+ out.println("</queue>");
+ out.println("<user name=\"user1\">");
+ out.println("<maxRunningApps>1</maxRunningApps>");
+ out.println("</user>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ // exceeds no limits
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1");
+ verifyAppRunnable(attId1, true);
+ verifyQueueNumRunnable("queue1", 1, 0);
+ // exceeds user limit
+ ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue2", "user1");
+ verifyAppRunnable(attId2, false);
+ verifyQueueNumRunnable("queue2", 0, 1);
+ // exceeds no limits
+ ApplicationAttemptId attId3 = createSchedulingRequest(1024, "queue1", "user2");
+ verifyAppRunnable(attId3, true);
+ verifyQueueNumRunnable("queue1", 2, 0);
+ // exceeds queue limit
+ ApplicationAttemptId attId4 = createSchedulingRequest(1024, "queue1", "user2");
+ verifyAppRunnable(attId4, false);
+ verifyQueueNumRunnable("queue1", 2, 1);
+
+ // Remove app 1 and both app 2 and app 4 should becomes runnable in its place
+ AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
+ attId1, RMAppAttemptState.FINISHED);
+ scheduler.handle(appRemovedEvent1);
+ verifyAppRunnable(attId2, true);
+ verifyQueueNumRunnable("queue2", 1, 0);
+ verifyAppRunnable(attId4, true);
+ verifyQueueNumRunnable("queue1", 2, 0);
+
+ // A new app to queue1 should not be runnable
+ ApplicationAttemptId attId5 = createSchedulingRequest(1024, "queue1", "user2");
+ verifyAppRunnable(attId5, false);
+ verifyQueueNumRunnable("queue1", 2, 1);
+ }
+
+ @Test
+ public void testMaxRunningAppsHierarchicalQueues() throws Exception {
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+ MockClock clock = new MockClock();
+ scheduler.setClock(clock);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queue1\">");
+ out.println(" <maxRunningApps>3</maxRunningApps>");
+ out.println(" <queue name=\"sub1\"></queue>");
+ out.println(" <queue name=\"sub2\"></queue>");
+ out.println(" <queue name=\"sub3\">");
+ out.println(" <maxRunningApps>1</maxRunningApps>");
+ out.println(" </queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ // exceeds no limits
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1");
+ verifyAppRunnable(attId1, true);
+ verifyQueueNumRunnable("queue1.sub1", 1, 0);
+ clock.tick(10);
+ // exceeds no limits
+ ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1.sub3", "user1");
+ verifyAppRunnable(attId2, true);
+ verifyQueueNumRunnable("queue1.sub3", 1, 0);
+ clock.tick(10);
+ // exceeds no limits
+ ApplicationAttemptId attId3 = createSchedulingRequest(1024, "queue1.sub2", "user1");
+ verifyAppRunnable(attId3, true);
+ verifyQueueNumRunnable("queue1.sub2", 1, 0);
+ clock.tick(10);
+ // exceeds queue1 limit
+ ApplicationAttemptId attId4 = createSchedulingRequest(1024, "queue1.sub2", "user1");
+ verifyAppRunnable(attId4, false);
+ verifyQueueNumRunnable("queue1.sub2", 1, 1);
+ clock.tick(10);
+ // exceeds sub3 limit
+ ApplicationAttemptId attId5 = createSchedulingRequest(1024, "queue1.sub3", "user1");
+ verifyAppRunnable(attId5, false);
+ verifyQueueNumRunnable("queue1.sub3", 1, 1);
+ clock.tick(10);
+
+ // Even though the app was removed from sub3, the app from sub2 gets to go
+ // because it came in first
+ AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
+ attId2, RMAppAttemptState.FINISHED);
+ scheduler.handle(appRemovedEvent1);
+ verifyAppRunnable(attId4, true);
+ verifyQueueNumRunnable("queue1.sub2", 2, 0);
+ verifyAppRunnable(attId5, false);
+ verifyQueueNumRunnable("queue1.sub3", 0, 1);
+
+ // Now test removal of a non-runnable app
+ AppRemovedSchedulerEvent appRemovedEvent2 = new AppRemovedSchedulerEvent(
+ attId5, RMAppAttemptState.KILLED);
+ scheduler.handle(appRemovedEvent2);
+ assertEquals(0, scheduler.maxRunningEnforcer.usersNonRunnableApps
+ .get("user1").size());
+ // verify app gone in queue accounting
+ verifyQueueNumRunnable("queue1.sub3", 0, 0);
+ // verify it doesn't become runnable when there would be space for it
+ AppRemovedSchedulerEvent appRemovedEvent3 = new AppRemovedSchedulerEvent(
+ attId4, RMAppAttemptState.FINISHED);
+ scheduler.handle(appRemovedEvent3);
+ verifyQueueNumRunnable("queue1.sub2", 1, 0);
+ verifyQueueNumRunnable("queue1.sub3", 0, 0);
}
@Test (timeout = 10000)
@@ -2493,23 +2650,23 @@ public class TestFairScheduler {
// Should get put into jerry
createSchedulingRequest(1024, "jerry", "someuser");
- assertEquals(1, jerryQueue.getAppSchedulables().size());
+ assertEquals(1, jerryQueue.getRunnableAppSchedulables().size());
// Should get forced into default
createSchedulingRequest(1024, "newqueue", "someuser");
- assertEquals(1, jerryQueue.getAppSchedulables().size());
- assertEquals(1, defaultQueue.getAppSchedulables().size());
+ assertEquals(1, jerryQueue.getRunnableAppSchedulables().size());
+ assertEquals(1, defaultQueue.getRunnableAppSchedulables().size());
// Would get put into someuser because of user-as-default-queue, but should
// be forced into default
createSchedulingRequest(1024, "default", "someuser");
- assertEquals(1, jerryQueue.getAppSchedulables().size());
- assertEquals(2, defaultQueue.getAppSchedulables().size());
+ assertEquals(1, jerryQueue.getRunnableAppSchedulables().size());
+ assertEquals(2, defaultQueue.getRunnableAppSchedulables().size());
// Should get put into jerry because of user-as-default-queue
createSchedulingRequest(1024, "default", "jerry");
- assertEquals(2, jerryQueue.getAppSchedulables().size());
- assertEquals(2, defaultQueue.getAppSchedulables().size());
+ assertEquals(2, jerryQueue.getRunnableAppSchedulables().size());
+ assertEquals(2, defaultQueue.getRunnableAppSchedulables().size());
}
@SuppressWarnings("resource")
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Mon Dec 2 17:41:44 2013
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
@@ -518,7 +519,7 @@ public class TestFifoScheduler {
public void testConcurrentAccessOnApplications() throws Exception {
FifoScheduler fs = new FifoScheduler();
TestCapacityScheduler.verifyConcurrentAccessOnApplications(
- fs.applications, FiCaSchedulerApp.class);
+ fs.applications, FiCaSchedulerApp.class, Queue.class);
}
@SuppressWarnings("resource")
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java Mon Dec 2 17:41:44 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.webproxy;
import java.io.IOException;
+import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -30,7 +31,6 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-
import org.apache.hadoop.fs.CommonConfigurationKeys;
public class WebAppProxy extends AbstractService {
@@ -89,7 +89,8 @@ public class WebAppProxy extends Abstrac
protected void serviceStart() throws Exception {
try {
proxyServer = new HttpServer.Builder().setName("proxy")
- .setBindAddress(bindAddress).setPort(port).setFindPort(port == 0)
+ .addEndpoint(URI.create("http://" + bindAddress + ":" + port))
+ .setFindPort(port == 0)
.setConf(getConfig()).setACL(acl).build();
proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java Mon Dec 2 17:41:44 2013
@@ -29,6 +29,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpCookie;
import java.net.HttpURLConnection;
+import java.net.URI;
import java.net.URL;
import java.util.List;
import java.util.Map;
@@ -126,7 +127,7 @@ public class TestWebAppProxyServlet {
proxy.init(configuration);
proxy.start();
- int proxyPort = proxy.proxy.proxyServer.getPort();
+ int proxyPort = proxy.proxy.proxyServer.getConnectorAddress(0).getPort();
AppReportFetcherForTest appReportFetcher = proxy.proxy.appReportFetcher;
// wrong url
@@ -285,8 +286,7 @@ public class TestWebAppProxyServlet {
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
proxyServer = new HttpServer.Builder()
.setName("proxy")
- .setBindAddress(bindAddress)
- .setPort(0)
+ .addEndpoint(URI.create("http://" + bindAddress + ":0"))
.setFindPort(true)
.setConf(conf)
.setACL(acl)
@@ -306,7 +306,7 @@ public class TestWebAppProxyServlet {
proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
proxyServer.start();
System.out.println("Proxy server is started at port " +
- proxyServer.getPort());
+ proxyServer.getConnectorAddress(0).getPort());
} catch (Exception e) {
LOG.fatal("Could not start proxy web server", e);
throw new YarnRuntimeException("Could not start proxy web server", e);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1547122&r1=1547121&r2=1547122&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Mon Dec 2 17:41:44 2013
@@ -287,6 +287,10 @@ Allocation file format
* primaryGroup: the app is placed into a queue with the name of the
primary group of the user who submitted it.
+ * secondaryGroupExistingQueue: the app is placed into a queue with a name
+ that matches a secondary group of the user who submitted it. The first
+ secondary group that matches a configured queue will be selected.
+
* default: the app is placed into the queue named "default".
* reject: the app is rejected.