You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by to...@apache.org on 2013/01/03 15:32:01 UTC
svn commit: r1428362 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/
hadoop-yarn/hadoop-yarn-server/hadoop-y...
Author: tomwhite
Date: Thu Jan 3 14:32:01 2013
New Revision: 1428362
URL: http://svn.apache.org/viewvc?rev=1428362&view=rev
Log:
YARN-288. Fair scheduler queue doesn't accept any jobs when ACLs are configured. Contributed by Sandy Ryza.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1428362&r1=1428361&r2=1428362&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Jan 3 14:32:01 2013
@@ -158,6 +158,9 @@ Release 2.0.3-alpha - Unreleased
YARN-192. Node update causes NPE in the fair scheduler.
(Sandy Ryza via tomwhite)
+ YARN-288. Fair scheduler queue doesn't accept any jobs when ACLs are
+ configured. (Sandy Ryza via tomwhite)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1428362&r1=1428361&r2=1428362&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java Thu Jan 3 14:32:01 2013
@@ -99,20 +99,6 @@ public class FSParentQueue extends FSQue
}
}
- public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
- synchronized (this) {
- if (getQueueAcls().get(acl).isUserAllowed(user)) {
- return true;
- }
- }
-
- if (parent != null) {
- return parent.hasAccess(acl, user);
- }
-
- return false;
- }
-
private synchronized QueueUserACLInfo getUserAclInfo(
UserGroupInformation user) {
QueueUserACLInfo userAclInfo =
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1428362&r1=1428361&r2=1428362&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Thu Jan 3 14:32:01 2013
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -118,6 +119,16 @@ public abstract class FSQueue extends Sc
return metrics;
}
+ public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
+ // Check if the leaf-queue allows access
+ if (queueMgr.getQueueAcls(getName()).get(acl).isUserAllowed(user)) {
+ return true;
+ }
+
+ // Check if parent-queue allows access
+ return parent != null && parent.hasAccess(acl, user);
+ }
+
/**
* Recomputes the fair shares for all queues and applications
* under this queue.
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1428362&r1=1428361&r2=1428362&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Thu Jan 3 14:32:01 2013
@@ -33,6 +33,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.SystemClock;
@@ -494,24 +495,15 @@ public class FairScheduler implements Re
new FSSchedulerApp(applicationAttemptId, user,
queue, new ActiveUsersManager(getRootQueueMetrics()),
rmContext);
-
- // Enforce ACLs
- UserGroupInformation userUgi;
- try {
- userUgi = UserGroupInformation.getCurrentUser();
- } catch (IOException ioe) {
- LOG.info("Failed to get current user information");
- return;
- }
- // Always a singleton list
- List<QueueUserACLInfo> info = queue.getQueueUserAclInfo(userUgi);
- if (!info.get(0).getUserAcls().contains(QueueACL.SUBMIT_APPLICATIONS)) {
+ // Enforce ACLs
+ UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user);
+ if (!queue.hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)) {
LOG.info("User " + userUgi.getUserName() +
- " cannot submit" + " applications to queue " + queue.getName());
+ " cannot submit applications to queue " + queue.getName());
return;
}
-
+
queue.addApp(schedulerApp);
queue.getMetrics().submitApp(user, applicationAttemptId.getAttemptId());
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1428362&r1=1428361&r2=1428362&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Thu Jan 3 14:32:01 2013
@@ -387,6 +387,16 @@ public class QueueManager {
queueMaxApps, userMaxApps, queueWeights, userMaxAppsDefault,
queueMaxAppsDefault, defaultSchedulingMode, minSharePreemptionTimeouts,
queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout);
+
+ // Root queue should have empty ACLs. As a queue's ACL is the union of
+ // its ACL and all its parents' ACLs, setting the roots' to empty will
+ // neither allow nor prohibit more access to its children.
+ Map<QueueACL, AccessControlList> rootAcls =
+ new HashMap<QueueACL, AccessControlList>();
+ rootAcls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(" "));
+ rootAcls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(" "));
+ queueAcls.put(ROOT_QUEUE, rootAcls);
+
for (String name: queueNamesInAllocFile) {
FSLeafQueue queue = getLeafQueue(name);
if (queueModes.containsKey(name)) {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1428362&r1=1428361&r2=1428362&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Thu Jan 3 14:32:01 2013
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -1243,4 +1245,34 @@ public class TestFairScheduler {
Assert.assertEquals(2, liveContainer.getContainer().getPriority().getPriority());
}
}
+
+ @Test
+ public void testAclSubmitApplication() throws Exception {
+ // Set acl's
+ Configuration conf = createConfiguration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queue1\">");
+ out.println("<aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ queueManager.initialize();
+
+ ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1",
+ "norealuserhasthisname", 1);
+ ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1",
+ "norealuserhasthisname2", 1);
+
+ FSSchedulerApp app1 = scheduler.applications.get(attId1);
+ assertNotNull("The application was not allowed", app1);
+ FSSchedulerApp app2 = scheduler.applications.get(attId2);
+ assertNull("The application was allowed", app2);
+ }
}