You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2017/10/20 04:46:04 UTC
[26/50] [abbrv] hadoop git commit: YARN-7308. TestApplicationACLs
fails with FairScheduler (rkanter)
YARN-7308. TestApplicationACLs fails with FairScheduler (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a615259
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a615259
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a615259
Branch: refs/heads/YARN-1011
Commit: 8a615259281511de639c1f9aa719a842e34a523f
Parents: 4540ffd
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Oct 16 15:34:32 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Mon Oct 16 15:34:32 2017 -0700
----------------------------------------------------------------------
.../ParameterizedSchedulerTestBase.java | 3 +-
.../resourcemanager/TestApplicationACLs.java | 90 +++++++++++++++-----
2 files changed, 69 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a615259/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
index 289ff1c..9a29a89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
@@ -89,7 +89,8 @@ public abstract class ParameterizedSchedulerTestBase {
}
}
- private void configureFairScheduler(YarnConfiguration conf) throws IOException {
+ protected void configureFairScheduler(YarnConfiguration conf)
+ throws IOException {
// Disable queueMaxAMShare limitation for fair scheduler
PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a615259/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index cab1679..c7ed02c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -22,7 +22,10 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Matchers.any;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
@@ -30,6 +33,9 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.junit.After;
import org.junit.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -57,19 +63,17 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-public class TestApplicationACLs {
+public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
private static final String APP_OWNER = "owner";
private static final String FRIEND = "friend";
@@ -82,27 +86,34 @@ public class TestApplicationACLs {
private static final Log LOG = LogFactory.getLog(TestApplicationACLs.class);
- static MockRM resourceManager;
- static Configuration conf = new YarnConfiguration();
- final static YarnRPC rpc = YarnRPC.create(conf);
- final static InetSocketAddress rmAddress = conf.getSocketAddr(
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
- private static ApplicationClientProtocol rmClient;
+ private MockRM resourceManager;
+ private Configuration conf;
+ private YarnRPC rpc;
+ private InetSocketAddress rmAddress;
+ private ApplicationClientProtocol rmClient;
+ private RecordFactory recordFactory;
+ private boolean isQueueUser;
- private static RecordFactory recordFactory = RecordFactoryProvider
- .getRecordFactory(conf);
-
- private static boolean isQueueUser = false;
+ public TestApplicationACLs(SchedulerType type) throws IOException {
+ super(type);
+ }
- @BeforeClass
- public static void setup() throws InterruptedException, IOException {
- RMStateStore store = RMStateStoreFactory.getStore(conf);
+ @Before
+ public void setup() throws InterruptedException, IOException {
+ conf = getConf();
+ rpc = YarnRPC.create(conf);
+ rmAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ RMStateStoreFactory.getStore(conf);
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
AccessControlList adminACL = new AccessControlList("");
adminACL.addGroup(SUPER_GROUP);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());
+ recordFactory = RecordFactoryProvider
+ .getRecordFactory(conf);
+ isQueueUser = false;
resourceManager = new MockRM(conf) {
@@ -161,13 +172,36 @@ public class TestApplicationACLs {
});
}
- @AfterClass
- public static void tearDown() {
+ @After
+ public void tearDown() {
if(resourceManager != null) {
resourceManager.stop();
}
}
+ @Override
+ protected void configureFairScheduler(YarnConfiguration conf)
+ throws IOException {
+ final String testDir = new File(System.getProperty("test.build.data",
+ "/tmp")).getAbsolutePath();
+ final String allocFile = new File(testDir, "test-queues.xml")
+ .getAbsolutePath();
+ PrintWriter out = new PrintWriter(new FileWriter(allocFile));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"root\" >");
+ out.println(" <queue name=\"default\">");
+ out.println(" </queue>");
+ out.println("</queue>");
+ out.println("<queuePlacementPolicy>");
+ out.println(" <rule name=\"specified\" create=\"false\" />");
+ out.println(" <rule name=\"reject\" />");
+ out.println("</queuePlacementPolicy>");
+ out.println("</allocations>");
+ out.close();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
+ }
+
@Test
public void testApplicationACLs() throws Exception {
@@ -207,6 +241,10 @@ public class TestApplicationACLs {
Resource resource = BuilderUtils.newResource(1024, 1);
context.setResource(resource);
amContainer.setApplicationACLs(acls);
+ if (conf.get(YarnConfiguration.RM_SCHEDULER)
+ .equals(FairScheduler.class.getName())) {
+ context.setQueue("root.default");
+ }
context.setAMContainerSpec(amContainer);
submitRequest.setApplicationSubmissionContext(context);
rmClient.submitApplication(submitRequest);
@@ -424,8 +462,14 @@ public class TestApplicationACLs {
GetApplicationReportResponse applicationReport =
rmClient.getApplicationReport(appReportRequest);
ApplicationReport appReport = applicationReport.getApplicationReport();
- Assert.assertTrue(appReport.getDiagnostics()
- .contains("submitted by user owner to unknown queue: InvalidQueue"));
+ if (conf.get(YarnConfiguration.RM_SCHEDULER)
+ .equals(FairScheduler.class.getName())) {
+ Assert.assertTrue(appReport.getDiagnostics()
+ .contains("Application rejected by queue placement policy"));
+ } else {
+ Assert.assertTrue(appReport.getDiagnostics()
+ .contains("submitted by user owner to unknown queue: InvalidQueue"));
+ }
}
private void verifyAdministerQueueUserAccess() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org