You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/02/27 19:10:42 UTC
[hadoop] branch trunk updated: YARN-10148. Add Unit test for queue
ACL for both FS and CS. Contributed by Kinga Marton
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 10461e0 YARN-10148. Add Unit test for queue ACL for both FS and CS. Contributed by Kinga Marton
10461e0 is described below
commit 10461e01932bcd82a9d4e3ab8109df7ead560b14
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Thu Feb 27 20:07:32 2020 +0100
YARN-10148. Add Unit test for queue ACL for both FS and CS. Contributed by Kinga Marton
---
.../yarn/server/resourcemanager/ACLsTestBase.java | 4 +
.../server/resourcemanager/QueueACLsTestBase.java | 146 +++++++++++++++++++++
.../capacity/TestCapacitySchedulerQueueACLs.java | 75 ++++++++++-
.../scheduler/fair/TestFairSchedulerQueueACLs.java | 62 ++++++++-
4 files changed, 283 insertions(+), 4 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
index ddebaaa..a011c8a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ACLsTestBase.java
@@ -113,5 +113,9 @@ public abstract class ACLsTestBase {
return userClient;
}
+ public Configuration getConf() {
+ return conf;
+ }
+
protected abstract Configuration createConfiguration() throws IOException;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java
index 82b3e24..ae9487e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java
@@ -18,9 +18,12 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.QueueACL;
import org.junit.Assert;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -43,6 +46,19 @@ import org.junit.Test;
public abstract class QueueACLsTestBase extends ACLsTestBase {
+ protected static final String QUEUED = "D";
+ protected static final String QUEUED1 = "D1";
+ private static final String ALL_ACL = "*";
+ private static final String NONE_ACL = " ";
+
+
+ abstract public String getQueueD();
+
+ abstract public String getQueueD1();
+
+ abstract public void updateConfigWithDAndD1Queues(String rootAcl,
+ String queueDAcl, String queueD1Acl) throws IOException;
+
@After
public void tearDown() {
if (resourceManager != null) {
@@ -75,6 +91,136 @@ public abstract class QueueACLsTestBase extends ACLsTestBase {
}
+ /**
+ * Test for the case when the following submit application
+ * and administer queue ACLs are defined:
+ * root: (none)
+ * D: * (all)
+ * D1: * (all)
+ * Expected result: the user will have access only to D and D1 queues.
+ * @throws IOException
+ */
+ @Test
+ public void testQueueAclRestrictedRootACL() throws IOException {
+ updateConfigWithDAndD1Queues(NONE_ACL, ALL_ACL, ALL_ACL);
+ checkAccess(false, true, true);
+ }
+
+ /**
+ * Test for the case when the following submit application
+ * and administer queue ACLs are defined:
+ * root: (none)
+ * D: (none)
+ * D1: (none)
+ * Expected result: the user will have to none of the queues.
+ * @throws IOException
+ */
+ @Test
+ public void testQueueAclNoAccess() throws IOException {
+ updateConfigWithDAndD1Queues(NONE_ACL, NONE_ACL, NONE_ACL);
+ checkAccess(false, false, false);
+ }
+
+ /**
+ * Test for the case when the following submit application
+ * and administer queue ACLs are defined:
+ * root: (none)
+ * D: * (all)
+ * D1: (none)
+ * Expected result: access to D1 will be permitted by root.D,
+ * so the user will be able to access queues D and D1.
+ * @throws IOException
+ */
+ @Test
+ public void testQueueAclRestrictedRootAndD1() throws IOException {
+ updateConfigWithDAndD1Queues(NONE_ACL, ALL_ACL, NONE_ACL);
+ checkAccess(false, true, true);
+ }
+
+ /**
+ * Test for the case when the following submit application
+ * and administer queue ACLs are defined:
+ * root: (none)
+ * D: (none)
+ * D1: (all)
+ * Expected result: only queue D1 can be accessed.
+ * @throws IOException
+ */
+ @Test
+ public void testQueueAclRestrictedRootAndD() throws IOException {
+ updateConfigWithDAndD1Queues(NONE_ACL, NONE_ACL, ALL_ACL);
+ checkAccess(false, false, true);
+ }
+
+ /**
+ * Test for the case when the following submit application
+ * and administer queue ACLs are defined:
+ * root: * (all)
+ * D: (none)
+ * D1: * (all)
+ * Expected result: access to D will be permitted from the root queue,
+ * so the user will be able to access queues root, D and D1.
+ * @throws IOException
+ */
+ @Test
+ public void testQueueAclRestrictedD() throws IOException {
+ updateConfigWithDAndD1Queues(ALL_ACL, NONE_ACL, ALL_ACL);
+ checkAccess(true, true, true);
+ }
+
+ /**
+ * Test for the case when the following submit application
+ * and administer queue ACLs are defined:
+ * root: * (all)
+ * D: * (all)
+ * D1: (none)
+ * Expected result: access to D1 will be permitted from queue D,
+ * so the user will be able to access queues root, D and D1.
+ * @throws IOException
+ */
+ @Test
+ public void testQueueAclRestrictedD1() throws IOException {
+ updateConfigWithDAndD1Queues(ALL_ACL, ALL_ACL, NONE_ACL);
+ checkAccess(true, true, true);
+ }
+
+ /**
+ * Test for the case when no ACLs are defined, so the default values are used
+ * Expected result: The default ACLs for the root queue is "*"(all) and for
+ * the other queues are " " (none), so the user will have access to all the
+ * queues because they will have permissions from the root.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testQueueAclDefaultValues() throws IOException {
+ updateConfigWithDAndD1Queues(null, null, null);
+ checkAccess(true, true, true);
+ }
+
+ private void checkAccess(boolean rootAccess, boolean dAccess,
+ boolean d1Access)throws IOException {
+ checkAccess(rootAccess, "root");
+ checkAccess(dAccess, getQueueD());
+ checkAccess(d1Access, getQueueD1());
+ }
+
+
+ private void checkAccess(boolean access, String queueName)
+ throws IOException {
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+
+ String failureMsg = "Wrong %s access to %s queue";
+ Assert.assertEquals(
+ String.format(failureMsg, QueueACL.ADMINISTER_QUEUE, queueName),
+ access, resourceManager.getResourceScheduler()
+ .checkAccess(user, QueueACL.ADMINISTER_QUEUE, queueName));
+ Assert.assertEquals(
+ String.format(failureMsg, QueueACL.SUBMIT_APPLICATIONS, queueName),
+ access, resourceManager.getResourceScheduler()
+ .checkAccess(user, QueueACL.SUBMIT_APPLICATIONS, queueName));
+ }
+
private void verifyGetClientAMToken(String submitter, String queueAdmin,
String queueName, boolean setupACLs) throws Exception {
ApplicationId applicationId =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
index 5feb94b..eeb2955 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -34,8 +35,10 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase {
csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {
QUEUEA, QUEUEB });
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEA, 50f);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + QUEUEB, 50f);
+ setQueueCapacity(csConf, 50,
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEA);
+ setQueueCapacity(csConf, 50,
+ CapacitySchedulerConfiguration.ROOT + "." + QUEUEB);
Map<QueueACL, AccessControlList> aclsOnQueueA =
new HashMap<QueueACL, AccessControlList>();
@@ -71,4 +74,72 @@ public class TestCapacitySchedulerQueueACLs extends QueueACLsTestBase {
return csConf;
}
+
+ @Override
+ public String getQueueD() {
+ return QUEUED;
+ }
+
+ @Override
+ public String getQueueD1() {
+ return QUEUED1;
+ }
+
+ /**
+ * Updates the configuration with the following queue hierarchy:
+ * root
+ * |
+ * D
+ * |
+ * D1.
+ * @param rootAcl administer queue and submit application ACL for root queue
+ * @param queueDAcl administer queue and submit application ACL for D queue
+ * @param queueD1Acl administer queue and submit application ACL for D1 queue
+ * @throws IOException
+ */
+ @Override
+ public void updateConfigWithDAndD1Queues(String rootAcl, String queueDAcl,
+ String queueD1Acl) throws IOException {
+ CapacitySchedulerConfiguration csConf =
+ (CapacitySchedulerConfiguration) getConf();
+ csConf.clear();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] {QUEUED, QUEUEA, QUEUEB});
+
+ String dPath = CapacitySchedulerConfiguration.ROOT + "." + QUEUED;
+ String d1Path = dPath + "." + QUEUED1;
+ csConf.setQueues(dPath, new String[] {QUEUED1});
+ setQueueCapacity(csConf, 100, d1Path);
+ setQueueCapacity(csConf, 30, CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEA);
+ setQueueCapacity(csConf, 50, CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEB);
+ setQueueCapacity(csConf, 20, dPath);
+
+ if (rootAcl != null) {
+ setAdminAndSubmitACL(csConf, rootAcl,
+ CapacitySchedulerConfiguration.ROOT);
+ }
+
+ if (queueDAcl != null) {
+ setAdminAndSubmitACL(csConf, queueDAcl, dPath);
+ }
+
+ if (queueD1Acl != null) {
+ setAdminAndSubmitACL(csConf, d1Path, queueD1Acl);
+ }
+ resourceManager.getResourceScheduler()
+ .reinitialize(csConf, resourceManager.getRMContext());
+ }
+
+ private void setQueueCapacity(CapacitySchedulerConfiguration csConf,
+ float capacity, String queuePath) {
+ csConf.setCapacity(queuePath, capacity);
+ }
+
+ private void setAdminAndSubmitACL(CapacitySchedulerConfiguration csConf,
+ String queueAcl, String queuePath) {
+ csConf.setAcl(queuePath, QueueACL.ADMINISTER_QUEUE, queueAcl);
+ csConf.setAcl(queuePath, QueueACL.SUBMIT_APPLICATIONS, queueAcl);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
index ad56a20..237f637 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.QueueACLsTestBase;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
@@ -32,8 +34,8 @@ public class TestFairSchedulerQueueACLs extends QueueACLsTestBase {
protected Configuration createConfiguration() {
FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
- final String testDir = new File(System.getProperty("test.build.data",
- "/tmp")).getAbsolutePath();
+ final String testDir = new File(System.getProperty(
+ GenericTestUtils.SYSPROP_TEST_DATA_DIR, "/tmp")).getAbsolutePath();
final String allocFile = new File(testDir, "test-queues.xml")
.getAbsolutePath();
@@ -57,4 +59,60 @@ public class TestFairSchedulerQueueACLs extends QueueACLsTestBase {
return fsConf;
}
+
+ @Override
+ public String getQueueD() {
+ return "root." + QUEUED;
+ }
+
+ @Override
+ public String getQueueD1() {
+ return "root."+ QUEUED + "." + QUEUED1;
+ }
+
+ /**
+ * Creates the following queue hierarchy:
+ * root
+ * |
+ * D
+ * |
+ * D1.
+ * @param rootAcl administer queue and submit application ACL for root queue
+ * @param queueDAcl administer queue and submit application ACL for D queue
+ * @param queueD1Acl administer queue and submit application ACL for D1 queue
+ * @throws IOException
+ */
+ @Override
+ public void updateConfigWithDAndD1Queues(String rootAcl, String queueDAcl,
+ String queueD1Acl) throws IOException {
+ FairSchedulerConfiguration fsConf = (FairSchedulerConfiguration) getConf();
+ fsConf.clear();
+ final String testDir = new File(System.getProperty(
+ GenericTestUtils.SYSPROP_TEST_DATA_DIR, "/tmp")).getAbsolutePath();
+ final String allocFile = new File(testDir, "test-queues.xml")
+ .getAbsolutePath();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(rootAcl)
+ .aclAdministerApps(rootAcl)
+ .subQueue(new AllocationFileQueue.Builder(QUEUED)
+ .aclAdministerApps(queueDAcl)
+ .aclSubmitApps(queueDAcl)
+ .subQueue(new AllocationFileQueue.Builder(QUEUED1)
+ .aclSubmitApps(queueD1Acl)
+ .aclAdministerApps(queueD1Acl)
+ .build())
+ .build())
+ .build())
+ .writeToFile(allocFile);
+
+ fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
+
+ fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ fsConf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
+ resourceManager.getResourceScheduler()
+ .reinitialize(fsConf, resourceManager.getRMContext());
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org