You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by wa...@apache.org on 2013/10/24 23:38:29 UTC
svn commit: r1535563 - in
/hadoop/common/branches/HDFS-4949/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/
...
Author: wang
Date: Thu Oct 24 21:38:25 2013
New Revision: 1535563
URL: http://svn.apache.org/r1535563
Log:
merge trunk into HDFS-4949 branch
Added:
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMAdminCLI.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMAdminCLI.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerQueueACLs.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
- copied unchanged from r1535559, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
Removed:
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestQueueACLs.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
Modified:
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt Thu Oct 24 21:38:25 2013
@@ -61,11 +61,12 @@ Release 2.3.0 - UNRELEASED
YARN-976. Document the meaning of a virtual core. (Sandy Ryza)
- YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza)
-
YARN-1182. MiniYARNCluster creates and inits the RM/NM only on start()
(Karthik Kambatla via Sandy Ryza)
+ HADOOP-9598. Improve code coverage of RMAdminCLI (Aleksey Gorshkov and
+ Andrey Klochkov via jeagles)
+
OPTIMIZATIONS
BUG FIXES
@@ -84,6 +85,12 @@ Release 2.3.0 - UNRELEASED
YARN-1300. SLS tests fail because conf puts YARN properties in
fair-scheduler.xml (Ted Yu via Sandy Ryza)
+ YARN-1183. MiniYARNCluster shutdown takes several minutes intermittently
+ (Andrey Klochkov via jeagles)
+
+ YARN-1305. RMHAProtocolService#serviceInit should handle HAUtil's
+ IllegalArgumentException (Tsuyoshi Ozawa via bikas)
+
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -95,6 +102,12 @@ Release 2.2.1 - UNRELEASED
YARN-305. Fair scheduler logs too many "Node offered to app" messages.
(Lohit Vijayarenu via Sandy Ryza)
+ YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza)
+
+ YARN-1288. Make Fair Scheduler ACLs more user friendly (Sandy Ryza)
+
+ YARN-1315. TestQueueACLs should also test FairScheduler (Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -120,6 +133,15 @@ Release 2.2.1 - UNRELEASED
YARN-1295. In UnixLocalWrapperScriptBuilder, using bash -c can cause Text
file busy errors (Sandy Ryza)
+ YARN-1185. Fixed FileSystemRMStateStore to not leave partial files that
+ prevent subsequent ResourceManager recovery. (Omkar Vinit Joshi via vinodkv)
+
+ YARN-1331. yarn.cmd exits with NoClassDefFoundError trying to run rmadmin or
+ logs. (cnauroth)
+
+ YARN-1330. Fair Scheduler: defaultQueueSchedulingPolicy does not take effect
+ (Sandy Ryza)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd Thu Oct 24 21:38:25 2013
@@ -149,7 +149,7 @@ goto :eof
goto :eof
:rmadmin
- set CLASS=org.apache.hadoop.yarn.server.resourcemanager.tools.RMAdmin
+ set CLASS=org.apache.hadoop.yarn.client.cli.RMAdminCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
@@ -200,7 +200,7 @@ goto :eof
goto :eof
:logs
- set CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper
+ set CLASS=org.apache.hadoop.yarn.client.cli.LogsCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
@@ -237,7 +237,6 @@ goto :eof
@echo where COMMAND is one of:
@echo resourcemanager run the ResourceManager
@echo nodemanager run a nodemanager on each slave
- @echo historyserver run job history servers as a standalone daemon
@echo rmadmin admin tools
@echo version print the version
@echo jar ^<jar^> run a jar file
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Thu Oct 24 21:38:25 2013
@@ -23,6 +23,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import java.util.Arrays;
@@ -42,10 +43,13 @@ public class HAUtil {
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.RM_WEBAPP_ADDRESS));
+ public static final String BAD_CONFIG_MESSAGE_PREFIX =
+ "Invalid configuration! ";
+
private HAUtil() { /* Hidden constructor */ }
private static void throwBadConfigurationException(String msg) {
- throw new YarnRuntimeException("Invalid configuration! " + msg);
+ throw new YarnRuntimeException(BAD_CONFIG_MESSAGE_PREFIX + msg);
}
/**
@@ -59,29 +63,137 @@ public class HAUtil {
YarnConfiguration.DEFAULT_RM_HA_ENABLED);
}
+ /**
+ * Verify configuration for Resource Manager HA.
+ * @param conf Configuration
+ * @throws YarnRuntimeException
+ */
+ public static void verifyAndSetConfiguration(Configuration conf)
+ throws YarnRuntimeException {
+ verifyAndSetRMHAIds(conf);
+ verifyAndSetRMHAId(conf);
+ verifyAndSetAllRpcAddresses(conf);
+ }
+
+
+ private static void verifyAndSetRMHAIds(Configuration conf) {
+ Collection<String> ids =
+ conf.getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS);
+ if (ids.size() <= 0) {
+ throwBadConfigurationException(
+ getInvalidValueMessage(YarnConfiguration.RM_HA_IDS,
+ conf.get(YarnConfiguration.RM_HA_IDS)));
+ } else if (ids.size() == 1) {
+ LOG.warn(getRMHAIdsWarningMessage(ids.toString()));
+ }
+
+ StringBuilder setValue = new StringBuilder();
+ for (String id: ids) {
+ setValue.append(id);
+ setValue.append(",");
+ }
+ conf.set(YarnConfiguration.RM_HA_IDS,
+ setValue.substring(0, setValue.length() - 1));
+ }
+
+ private static void verifyAndSetRMHAId(Configuration conf) {
+ String rmId = conf.getTrimmed(YarnConfiguration.RM_HA_ID);
+ if (rmId == null) {
+ throwBadConfigurationException(
+ getNeedToSetValueMessage(YarnConfiguration.RM_HA_ID));
+ } else {
+ Collection<String> ids = getRMHAIds(conf);
+ if (!ids.contains(rmId)) {
+ throwBadConfigurationException(
+ getRMHAIdNeedToBeIncludedMessage(ids.toString(), rmId));
+ }
+ }
+ conf.set(YarnConfiguration.RM_HA_ID, rmId);
+ }
+
+ private static void verifyAndSetConfValue(String prefix, Configuration conf) {
+ String confKey = null;
+ String confValue = null;
+ try {
+ confKey = getConfKeyForRMInstance(prefix, conf);
+ confValue = getConfValueForRMInstance(prefix, conf);
+ conf.set(prefix, confValue);
+ } catch (YarnRuntimeException yre) {
+ // Error at getRMHAId()
+ throw yre;
+ } catch (IllegalArgumentException iae) {
+ String errmsg;
+ if (confKey == null) {
+ // Error at addSuffix
+ errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID,
+ getRMHAId(conf));
+ } else {
+ // Error at Configuration#set.
+ errmsg = getNeedToSetValueMessage(confKey);
+ }
+ throwBadConfigurationException(errmsg);
+ }
+ }
+
+ public static void verifyAndSetAllRpcAddresses(Configuration conf) {
+ for (String confKey : RPC_ADDRESS_CONF_KEYS) {
+ verifyAndSetConfValue(confKey, conf);
+ }
+ }
+
+ /**
+ * @param conf Configuration. Please use getRMHAIds to check.
+ * @return RM Ids on success
+ */
public static Collection<String> getRMHAIds(Configuration conf) {
- return conf.getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS);
+ return conf.getStringCollection(YarnConfiguration.RM_HA_IDS);
}
/**
- * @param conf Configuration
+ * @param conf Configuration. Please use verifyAndSetRMHAId to check.
* @return RM Id on success
- * @throws YarnRuntimeException for configurations without a node id
*/
@VisibleForTesting
- public static String getRMHAId(Configuration conf) {
- String rmId = conf.get(YarnConfiguration.RM_HA_ID);
- if (rmId == null) {
- throwBadConfigurationException(YarnConfiguration.RM_HA_ID +
- " needs to be set in a HA configuration");
- }
- return rmId;
+ static String getRMHAId(Configuration conf) {
+ return conf.get(YarnConfiguration.RM_HA_ID);
+ }
+
+ @VisibleForTesting
+ static String getNeedToSetValueMessage(String confKey) {
+ return confKey + " needs to be set in a HA configuration.";
+ }
+
+ @VisibleForTesting
+ static String getInvalidValueMessage(String confKey,
+ String invalidValue){
+ return "Invalid value of " + confKey +". "
+ + "Current value is " + invalidValue;
+ }
+
+ @VisibleForTesting
+ static String getRMHAIdNeedToBeIncludedMessage(String ids,
+ String rmId) {
+ return YarnConfiguration.RM_HA_IDS + "("
+ + ids + ") need to contain " + YarnConfiguration.RM_HA_ID + "("
+ + rmId + ") in a HA configuration.";
+ }
+
+ @VisibleForTesting
+ static String getRMHAIdsWarningMessage(String ids) {
+ return "Resource Manager HA is enabled, but " +
+ YarnConfiguration.RM_HA_IDS + " has only one id(" +
+ ids.toString() + ")";
+ }
+
+ private static String getConfKeyForRMInstance(String prefix,
+ Configuration conf) {
+ return addSuffix(prefix, getRMHAId(conf));
}
private static String getConfValueForRMInstance(String prefix,
Configuration conf) {
- String confKey = addSuffix(prefix, getRMHAId(conf));
- String retVal = conf.get(confKey);
+ String confKey = getConfKeyForRMInstance(prefix, conf);
+ String retVal = conf.getTrimmed(confKey);
if (LOG.isTraceEnabled()) {
LOG.trace("getConfValueForRMInstance: prefix = " + prefix +
"; confKey being looked up = " + confKey +
@@ -96,16 +208,6 @@ public class HAUtil {
return (value == null) ? defaultValue : value;
}
- private static void setConfValue(String prefix, Configuration conf) {
- conf.set(prefix, getConfValueForRMInstance(prefix, conf));
- }
-
- public static void setAllRpcAddresses(Configuration conf) {
- for (String confKey : RPC_ADDRESS_CONF_KEYS) {
- setConfValue(confKey, conf);
- }
- }
-
/** Add non empty and non null suffix to a key */
@VisibleForTesting
public static String addSuffix(String key, String suffix) {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java Thu Oct 24 21:38:25 2013
@@ -162,7 +162,7 @@ public class RMAdminCLI extends Configur
}
}
- private ResourceManagerAdministrationProtocol createAdminProtocol() throws IOException {
+ protected ResourceManagerAdministrationProtocol createAdminProtocol() throws IOException {
// Get the current configuration
final YarnConfiguration conf = new YarnConfiguration(getConf());
return ClientRMProxy.createRMProxy(conf, ResourceManagerAdministrationProtocol.class);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java Thu Oct 24 21:38:25 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.conf;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.junit.Before;
import org.junit.Test;
@@ -27,53 +28,134 @@ import org.junit.Test;
import java.util.Collection;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
public class TestHAUtil {
private Configuration conf;
- private static final String RM1_ADDRESS = "1.2.3.4:8021";
+ private static final String RM1_ADDRESS_UNTRIMMED = " \t\t\n 1.2.3.4:8021 \n\t ";
+ private static final String RM1_ADDRESS = RM1_ADDRESS_UNTRIMMED.trim();
private static final String RM2_ADDRESS = "localhost:8022";
- private static final String RM1_NODE_ID = "rm1";
+ private static final String RM1_NODE_ID_UNTRIMMED = "rm1 ";
+ private static final String RM1_NODE_ID = RM1_NODE_ID_UNTRIMMED.trim();
private static final String RM2_NODE_ID = "rm2";
+ private static final String RM3_NODE_ID = "rm3";
+ private static final String RM_INVALID_NODE_ID = ".rm";
+ private static final String RM_NODE_IDS_UNTRIMMED = RM1_NODE_ID_UNTRIMMED + "," + RM2_NODE_ID;
+ private static final String RM_NODE_IDS = RM1_NODE_ID + "," + RM2_NODE_ID;
@Before
public void setUp() {
conf = new Configuration();
- conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
- conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
+ conf.set(YarnConfiguration.RM_HA_IDS, RM_NODE_IDS_UNTRIMMED);
+ conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
- conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
+ // configuration key itself cannot contains space/tab/return chars.
+ conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
}
}
@Test
public void testGetRMServiceId() throws Exception {
+ conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
Collection<String> rmhaIds = HAUtil.getRMHAIds(conf);
assertEquals(2, rmhaIds.size());
+
+ String[] ids = rmhaIds.toArray(new String[0]);
+ assertEquals(RM1_NODE_ID, ids[0]);
+ assertEquals(RM2_NODE_ID, ids[1]);
}
@Test
public void testGetRMId() throws Exception {
+ conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
assertEquals("Does not honor " + YarnConfiguration.RM_HA_ID,
- RM1_NODE_ID, HAUtil.getRMHAId(conf));
- conf = new YarnConfiguration();
- try {
- HAUtil.getRMHAId(conf);
- fail("getRMHAId() fails to throw an exception when RM_HA_ID is not set");
- } catch (YarnRuntimeException yre) {
- // do nothing
- }
+ RM1_NODE_ID, HAUtil.getRMHAId(conf));
+
+ conf.clear();
+ assertNull("Return null when " + YarnConfiguration.RM_HA_ID
+ + " is not set", HAUtil.getRMHAId(conf));
}
@Test
- public void testSetGetRpcAddresses() throws Exception {
- HAUtil.setAllRpcAddresses(conf);
+ public void testVerifyAndSetConfiguration() throws Exception {
+ try {
+ HAUtil.verifyAndSetConfiguration(conf);
+ } catch (YarnRuntimeException e) {
+ fail("Should not throw any exceptions.");
+ }
+
+ assertEquals("Should be saved as Trimmed collection",
+ StringUtils.getStringCollection(RM_NODE_IDS), HAUtil.getRMHAIds(conf));
+ assertEquals("Should be saved as Trimmed string",
+ RM1_NODE_ID, HAUtil.getRMHAId(conf));
for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
assertEquals("RPC address not set for " + confKey,
- RM1_ADDRESS, conf.get(confKey));
+ RM1_ADDRESS, conf.get(confKey));
+ }
+
+ conf.clear();
+ conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID);
+ try {
+ HAUtil.verifyAndSetConfiguration(conf);
+ } catch (YarnRuntimeException e) {
+ assertEquals("YarnRuntimeException by getRMId()",
+ HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
+ HAUtil.getNeedToSetValueMessage(YarnConfiguration.RM_HA_ID),
+ e.getMessage());
+ }
+
+ conf.clear();
+ conf.set(YarnConfiguration.RM_HA_ID, RM_INVALID_NODE_ID);
+ conf.set(YarnConfiguration.RM_HA_IDS, RM_INVALID_NODE_ID);
+ for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ // simulate xml with invalid node id
+ conf.set(confKey + RM_INVALID_NODE_ID, RM_INVALID_NODE_ID);
+ }
+ try {
+ HAUtil.verifyAndSetConfiguration(conf);
+ } catch (YarnRuntimeException e) {
+ assertEquals("YarnRuntimeException by addSuffix()",
+ HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
+ HAUtil.getInvalidValueMessage(YarnConfiguration.RM_HA_ID,
+ RM_INVALID_NODE_ID),
+ e.getMessage());
+ }
+
+ conf.clear();
+ // simulate the case HAUtil.RPC_ADDRESS_CONF_KEYS are not set
+ conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
+ conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID);
+ try {
+ HAUtil.verifyAndSetConfiguration(conf);
+ fail("Should throw YarnRuntimeException. by Configuration#set()");
+ } catch (YarnRuntimeException e) {
+ String confKey =
+ HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID);
+ assertEquals("YarnRuntimeException by Configuration#set()",
+ HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(confKey),
+ e.getMessage());
+ }
+
+ // simulate the case YarnConfiguration.RM_HA_IDS doesn't contain
+ // the value of YarnConfiguration.RM_HA_ID
+ conf.clear();
+ conf.set(YarnConfiguration.RM_HA_IDS, RM2_NODE_ID + "," + RM3_NODE_ID);
+ conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID_UNTRIMMED);
+ for (String confKey : HAUtil.RPC_ADDRESS_CONF_KEYS) {
+ conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS_UNTRIMMED);
+ conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
+ }
+ try {
+ HAUtil.verifyAndSetConfiguration(conf);
+ } catch (YarnRuntimeException e) {
+ assertEquals("YarnRuntimeException by getRMId()'s validation",
+ HAUtil.BAD_CONFIG_MESSAGE_PREFIX +
+ HAUtil.getRMHAIdNeedToBeIncludedMessage("[rm2, rm3]", RM1_NODE_ID),
+ e.getMessage());
}
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMHAProtocolService.java Thu Oct 24 21:38:25 2013
@@ -57,7 +57,7 @@ public class RMHAProtocolService extends
this.conf = conf;
haEnabled = HAUtil.isHAEnabled(this.conf);
if (haEnabled) {
- HAUtil.setAllRpcAddresses(this.conf);
+ HAUtil.verifyAndSetConfiguration(conf);
rm.setConf(this.conf);
}
rm.createAndInitActiveServices();
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java Thu Oct 24 21:38:25 2013
@@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -118,6 +119,9 @@ public class FileSystemRMStateStore exte
for (FileStatus childNodeStatus : fs.listStatus(appDir.getPath())) {
assert childNodeStatus.isFile();
String childNodeName = childNodeStatus.getPath().getName();
+ if (checkAndRemovePartialRecord(childNodeStatus.getPath())) {
+ continue;
+ }
byte[] childData =
readFile(childNodeStatus.getPath(), childNodeStatus.getLen());
if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
@@ -178,12 +182,28 @@ public class FileSystemRMStateStore exte
}
}
+ private boolean checkAndRemovePartialRecord(Path record) throws IOException {
+ // If the file ends with .tmp then it shows that it failed
+ // during saving state into state store. The file will be deleted as a
+ // part of this call
+ if (record.getName().endsWith(".tmp")) {
+ LOG.error("incomplete rm state store entry found :"
+ + record);
+ fs.delete(record, false);
+ return true;
+ }
+ return false;
+ }
+
private void loadRMDTSecretManagerState(RMState rmState) throws Exception {
FileStatus[] childNodes = fs.listStatus(rmDTSecretManagerRoot);
for(FileStatus childNodeStatus : childNodes) {
assert childNodeStatus.isFile();
String childNodeName = childNodeStatus.getPath().getName();
+ if (checkAndRemovePartialRecord(childNodeStatus.getPath())) {
+ continue;
+ }
if(childNodeName.startsWith(DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX)) {
rmState.rmSecretManagerState.dtSequenceNumber =
Integer.parseInt(childNodeName.split("_")[1]);
@@ -344,10 +364,19 @@ public class FileSystemRMStateStore exte
return data;
}
+ /*
+ * In order to make this write atomic as a part of write we will first write
+ * data to .tmp file and then rename it. Here we are assuming that rename is
+ * atomic for underlying file system.
+ */
private void writeFile(Path outputPath, byte[] data) throws Exception {
- FSDataOutputStream fsOut = fs.create(outputPath, false);
+ Path tempPath =
+ new Path(outputPath.getParent(), outputPath.getName() + ".tmp");
+ FSDataOutputStream fsOut = null;
+ fsOut = fs.create(tempPath, false);
fsOut.write(data);
fsOut.close();
+ fs.rename(tempPath, outputPath);
}
private boolean renameFile(Path src, Path dst) throws Exception {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java Thu Oct 24 21:38:25 2013
@@ -19,12 +19,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
@@ -45,12 +43,6 @@ public interface Queue {
QueueMetrics getMetrics();
/**
- * Get ACLs for the queue.
- * @return ACLs for the queue
- */
- public Map<QueueACL, AccessControlList> getQueueAcls();
-
- /**
* Get queue information
* @param includeChildQueues include child queues?
* @param recursive recursively get child queue information?
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Thu Oct 24 21:38:25 2013
@@ -527,11 +527,6 @@ public class LeafQueue implements CSQueu
}
@Override
- public synchronized Map<QueueACL, AccessControlList> getQueueAcls() {
- return new HashMap<QueueACL, AccessControlList>(acls);
- }
-
- @Override
public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
queueInfo.setCurrentCapacity(usedCapacity);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Thu Oct 24 21:38:25 2013
@@ -300,11 +300,6 @@ public class ParentQueue implements CSQu
}
@Override
- public synchronized Map<QueueACL, AccessControlList> getQueueAcls() {
- return new HashMap<QueueACL, AccessControlList>(acls);
- }
-
- @Override
public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
queueInfo.setCurrentCapacity(usedCapacity);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Thu Oct 24 21:38:25 2013
@@ -24,14 +24,12 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -177,8 +175,7 @@ public class FSLeafQueue extends FSQueue
recordFactory.newRecordInstance(QueueUserACLInfo.class);
List<QueueACL> operations = new ArrayList<QueueACL>();
for (QueueACL operation : QueueACL.values()) {
- Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
- if (acls.get(operation).isUserAllowed(user)) {
+ if (hasAccess(operation, user)) {
operations.add(operation);
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Thu Oct 24 21:38:25 2013
@@ -20,13 +20,10 @@ package org.apache.hadoop.yarn.server.re
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -136,12 +133,6 @@ public abstract class FSQueue extends Sc
}
@Override
- public Map<QueueACL, AccessControlList> getQueueAcls() {
- Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
- return new HashMap<QueueACL, AccessControlList>(acls);
- }
-
- @Override
public FSQueueMetrics getMetrics() {
return metrics;
}
@@ -154,7 +145,7 @@ public abstract class FSQueue extends Sc
public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
// Check if the leaf-queue allows access
- if (queueMgr.getQueueAcls(getName()).get(acl).isUserAllowed(user)) {
+ if (queueMgr.getQueueAcl(getName(), acl).isUserAllowed(user)) {
return true;
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java Thu Oct 24 21:38:25 2013
@@ -50,7 +50,7 @@ public class FairSchedulerConfiguration
private static final String CONF_PREFIX = "yarn.scheduler.fair.";
- protected static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
+ public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
protected static final String EVENT_LOG_DIR = "eventlog.dir";
@@ -113,6 +113,10 @@ public class FairSchedulerConfiguration
protected static final String MAX_ASSIGN = CONF_PREFIX + "max.assign";
protected static final int DEFAULT_MAX_ASSIGN = -1;
+ public FairSchedulerConfiguration() {
+ super();
+ }
+
public FairSchedulerConfiguration(Configuration conf) {
super(conf);
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Thu Oct 24 21:38:25 2013
@@ -72,6 +72,9 @@ public class QueueManager {
* (this is done to prevent loading a file that hasn't been fully written).
*/
public static final long ALLOC_RELOAD_WAIT = 5 * 1000;
+
+ private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*");
+ private static final AccessControlList NOBODY_ACL = new AccessControlList(" ");
private final FairScheduler scheduler;
@@ -375,31 +378,24 @@ public class QueueManager {
queueMaxAppsDefault, defaultSchedPolicy, minSharePreemptionTimeouts,
queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout);
- // Update metrics
+ // Make sure all queues exist
+ for (String name: queueNamesInAllocFile) {
+ getLeafQueue(name, true);
+ }
+
for (FSQueue queue : queues.values()) {
+ // Update queue metrics
FSQueueMetrics queueMetrics = queue.getMetrics();
queueMetrics.setMinShare(queue.getMinShare());
queueMetrics.setMaxShare(queue.getMaxShare());
+ // Set scheduling policies
+ if (queuePolicies.containsKey(queue.getName())) {
+ queue.setPolicy(queuePolicies.get(queue.getName()));
+ } else {
+ queue.setPolicy(SchedulingPolicy.getDefault());
+ }
}
-
- // Root queue should have empty ACLs. As a queue's ACL is the union of
- // its ACL and all its parents' ACLs, setting the roots' to empty will
- // neither allow nor prohibit more access to its children.
- Map<QueueACL, AccessControlList> rootAcls =
- new HashMap<QueueACL, AccessControlList>();
- rootAcls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(" "));
- rootAcls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(" "));
- queueAcls.put(ROOT_QUEUE, rootAcls);
- // Create all queus
- for (String name: queueNamesInAllocFile) {
- getLeafQueue(name, true);
- }
-
- // Set custom policies as specified
- for (Map.Entry<String, SchedulingPolicy> entry : queuePolicies.entrySet()) {
- queues.get(entry.getKey()).setPolicy(entry.getValue());
- }
}
}
@@ -454,10 +450,10 @@ public class QueueManager {
policy.initialize(scheduler.getClusterCapacity());
queuePolicies.put(queueName, policy);
} else if ("aclSubmitApps".equals(field.getTagName())) {
- String text = ((Text)field.getFirstChild()).getData().trim();
+ String text = ((Text)field.getFirstChild()).getData();
acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text));
} else if ("aclAdministerApps".equals(field.getTagName())) {
- String text = ((Text)field.getFirstChild()).getData().trim();
+ String text = ((Text)field.getFirstChild()).getData();
acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text));
} else if ("queue".endsWith(field.getTagName()) ||
"pool".equals(field.getTagName())) {
@@ -577,21 +573,16 @@ public class QueueManager {
/**
* Get the ACLs associated with this queue. If a given ACL is not explicitly
- * configured, include the default value for that ACL.
- */
- public Map<QueueACL, AccessControlList> getQueueAcls(String queue) {
- HashMap<QueueACL, AccessControlList> out = new HashMap<QueueACL, AccessControlList>();
- Map<QueueACL, AccessControlList> queueAcl = info.queueAcls.get(queue);
- if (queueAcl != null) {
- out.putAll(queueAcl);
- }
- if (!out.containsKey(QueueACL.ADMINISTER_QUEUE)) {
- out.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList("*"));
- }
- if (!out.containsKey(QueueACL.SUBMIT_APPLICATIONS)) {
- out.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList("*"));
+ * configured, include the default value for that ACL. The default for the
+ * root queue is everybody ("*") and the default for all other queues is
+ * nobody ("")
+ */
+ public AccessControlList getQueueAcl(String queue, QueueACL operation) {
+ Map<QueueACL, AccessControlList> queueAcls = info.queueAcls.get(queue);
+ if (queueAcls == null || !queueAcls.containsKey(operation)) {
+ return (queue.equals(ROOT_QUEUE)) ? EVERYBODY_ACL : NOBODY_ACL;
}
- return out;
+ return queueAcls.get(operation);
}
static class QueueManagerInfo {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Thu Oct 24 21:38:25 2013
@@ -156,7 +156,6 @@ public class FifoScheduler implements Re
return queueInfo;
}
- @Override
public Map<QueueACL, AccessControlList> getQueueAcls() {
Map<QueueACL, AccessControlList> acls =
new HashMap<QueueACL, AccessControlList>();
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java Thu Oct 24 21:38:25 2013
@@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.TestRMStateStore.TestDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreTestBase.TestDispatcher;
import org.apache.hadoop.util.ZKUtil;
import org.apache.zookeeper.CreateMode;
@@ -43,17 +43,20 @@ import static org.junit.Assert.fail;
public class TestZKRMStateStoreZKClientConnections extends
ClientBaseWithFixes {
+
private static final int ZK_OP_WAIT_TIME = 3000;
private Log LOG =
LogFactory.getLog(TestZKRMStateStoreZKClientConnections.class);
class TestZKClient {
+
ZKRMStateStore store;
boolean forExpire = false;
TestForwardingWatcher watcher;
CyclicBarrier syncBarrier = new CyclicBarrier(2);
protected class TestZKRMStateStore extends ZKRMStateStore {
+
public TestZKRMStateStore(Configuration conf, String workingZnode)
throws Exception {
init(conf);
@@ -87,6 +90,7 @@ public class TestZKRMStateStoreZKClientC
private class TestForwardingWatcher extends
ClientBaseWithFixes.CountdownWatcher {
+
public void process(WatchedEvent event) {
super.process(event);
try {
@@ -187,7 +191,7 @@ public class TestZKRMStateStoreZKClientC
}
}
- @Test (timeout = 20000)
+ @Test(timeout = 20000)
public void testSetZKAcl() {
TestZKClient zkClientTester = new TestZKClient();
YarnConfiguration conf = new YarnConfiguration();
@@ -196,10 +200,11 @@ public class TestZKRMStateStoreZKClientC
zkClientTester.store.zkClient.delete(zkClientTester.store
.znodeWorkingPath, -1);
fail("Shouldn't be able to delete path");
- } catch (Exception e) {/* expected behavior */}
+ } catch (Exception e) {/* expected behavior */
+ }
}
- @Test (timeout = 20000)
+ @Test(timeout = 20000)
public void testInvalidZKAclConfiguration() {
TestZKClient zkClientTester = new TestZKClient();
YarnConfiguration conf = new YarnConfiguration();
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Thu Oct 24 21:38:25 2013
@@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
@@ -807,6 +808,7 @@ public class TestFairScheduler {
out.println("<queue name=\"queueB\">");
out.println("<minResources>2048mb,0vcores</minResources>");
out.println("<aclAdministerApps>alice,bob admins</aclAdministerApps>");
+ out.println("<schedulingPolicy>fair</schedulingPolicy>");
out.println("</queue>");
// Give queue C no minimum
out.println("<queue name=\"queueC\">");
@@ -833,6 +835,8 @@ public class TestFairScheduler {
+ "</defaultMinSharePreemptionTimeout>");
// Set fair share preemption timeout to 5 minutes
out.println("<fairSharePreemptionTimeout>300</fairSharePreemptionTimeout>");
+ // Set default scheduling policy to DRF
+ out.println("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
out.println("</allocations>");
out.close();
@@ -865,22 +869,25 @@ public class TestFairScheduler {
assertEquals(10, queueManager.getUserMaxApps("user1"));
assertEquals(5, queueManager.getUserMaxApps("user2"));
+ // Root should get * ACL
+ assertEquals("*",queueManager.getQueueAcl("root",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+ assertEquals("*", queueManager.getQueueAcl("root",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
+
// Unspecified queues should get default ACL
- Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("root.queueA");
- assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
- assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
- assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
- assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
+ assertEquals(" ",queueManager.getQueueAcl("root.queueA",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+ assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue B ACL
- Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
- assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
- assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
-
- // Queue c ACL
- Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
- assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
- assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
+ assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueB",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+
+ // Queue C ACL
+ assertEquals("alice,bob admins",queueManager.getQueueAcl("root.queueC",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME));
@@ -891,6 +898,18 @@ public class TestFairScheduler {
assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root.queueA"));
assertEquals(60000, queueManager.getMinSharePreemptionTimeout("root.queueE"));
assertEquals(300000, queueManager.getFairSharePreemptionTimeout());
+
+ // Verify existing queues have default scheduling policy
+ assertEquals(DominantResourceFairnessPolicy.NAME,
+ queueManager.getQueue("root").getPolicy().getName());
+ assertEquals(DominantResourceFairnessPolicy.NAME,
+ queueManager.getQueue("root.queueA").getPolicy().getName());
+ // Verify default is overriden if specified explicitly
+ assertEquals(FairSharePolicy.NAME,
+ queueManager.getQueue("root.queueB").getPolicy().getName());
+ // Verify new queue gets default scheduling policy
+ assertEquals(DominantResourceFairnessPolicy.NAME,
+ queueManager.getLeafQueue("root.newqueue", true).getPolicy().getName());
}
@Test
@@ -1063,21 +1082,19 @@ public class TestFairScheduler {
assertEquals(5, queueManager.getUserMaxApps("user2"));
// Unspecified queues should get default ACL
- Map<QueueACL, AccessControlList> aclsA = queueManager.getQueueAcls("queueA");
- assertTrue(aclsA.containsKey(QueueACL.ADMINISTER_QUEUE));
- assertEquals("*", aclsA.get(QueueACL.ADMINISTER_QUEUE).getAclString());
- assertTrue(aclsA.containsKey(QueueACL.SUBMIT_APPLICATIONS));
- assertEquals("*", aclsA.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
+ assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+ assertEquals(" ", queueManager.getQueueAcl("root.queueA",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
// Queue B ACL
- Map<QueueACL, AccessControlList> aclsB = queueManager.getQueueAcls("root.queueB");
- assertTrue(aclsB.containsKey(QueueACL.ADMINISTER_QUEUE));
- assertEquals("alice,bob admins", aclsB.get(QueueACL.ADMINISTER_QUEUE).getAclString());
-
- // Queue c ACL
- Map<QueueACL, AccessControlList> aclsC = queueManager.getQueueAcls("root.queueC");
- assertTrue(aclsC.containsKey(QueueACL.SUBMIT_APPLICATIONS));
- assertEquals("alice,bob admins", aclsC.get(QueueACL.SUBMIT_APPLICATIONS).getAclString());
+ assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueB",
+ QueueACL.ADMINISTER_QUEUE).getAclString());
+
+ // Queue C ACL
+ assertEquals("alice,bob admins", queueManager.getQueueAcl("root.queueC",
+ QueueACL.SUBMIT_APPLICATIONS).getAclString());
+
assertEquals(120000, queueManager.getMinSharePreemptionTimeout("root." +
YarnConfiguration.DEFAULT_QUEUE_NAME));
@@ -1664,9 +1681,13 @@ public class TestFairScheduler {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
- out.println("<aclAdministerApps>norealuserhasthisname</aclAdministerApps>");
+ out.println("<queue name=\"root\">");
+ out.println(" <aclSubmitApps> </aclSubmitApps>");
+ out.println(" <aclAdministerApps> </aclAdministerApps>");
+ out.println(" <queue name=\"queue1\">");
+ out.println(" <aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
+ out.println(" <aclAdministerApps>norealuserhasthisname</aclAdministerApps>");
+ out.println(" </queue>");
out.println("</queue>");
out.println("</allocations>");
out.close();
@@ -1893,9 +1914,13 @@ public class TestFairScheduler {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<aclSubmitApps>userallow</aclSubmitApps>");
- out.println("<aclAdministerApps>userallow</aclAdministerApps>");
+ out.println("<queue name=\"root\">");
+ out.println(" <aclSubmitApps> </aclSubmitApps>");
+ out.println(" <aclAdministerApps> </aclAdministerApps>");
+ out.println(" <queue name=\"queue1\">");
+ out.println(" <aclSubmitApps>userallow</aclSubmitApps>");
+ out.println(" <aclAdministerApps>userallow</aclAdministerApps>");
+ out.println(" </queue>");
out.println("</queue>");
out.println("</allocations>");
out.close();
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Thu Oct 24 21:38:25 2013
@@ -22,6 +22,8 @@ import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -34,8 +36,10 @@ import org.apache.hadoop.service.Abstrac
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -52,6 +56,10 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
/**
@@ -83,6 +91,9 @@ public class MiniYARNCluster extends Com
private ResourceManagerWrapper resourceManagerWrapper;
+ private ConcurrentMap<ApplicationAttemptId, Long> appMasters =
+ new ConcurrentHashMap<ApplicationAttemptId, Long>(16, 0.75f, 2);
+
private File testWorkDir;
// Number of nm-local-dirs per nodemanager
@@ -210,6 +221,16 @@ public class MiniYARNCluster extends Com
};
};
resourceManager.init(conf);
+ resourceManager.getRMContext().getDispatcher().register(RMAppAttemptEventType.class,
+ new EventHandler<RMAppAttemptEvent>() {
+ public void handle(RMAppAttemptEvent event) {
+ if (event instanceof RMAppAttemptRegistrationEvent) {
+ appMasters.put(event.getApplicationAttemptId(), event.getTimestamp());
+ } else if (event instanceof RMAppAttemptUnregistrationEvent) {
+ appMasters.remove(event.getApplicationAttemptId());
+ }
+ }
+ });
super.serviceInit(conf);
}
@@ -243,9 +264,22 @@ public class MiniYARNCluster extends Com
WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
}
+ private void waitForAppMastersToFinish(long timeoutMillis) throws InterruptedException {
+ long started = System.currentTimeMillis();
+ synchronized (appMasters) {
+ while (!appMasters.isEmpty() && System.currentTimeMillis() - started < timeoutMillis) {
+ appMasters.wait(1000);
+ }
+ }
+ if (!appMasters.isEmpty()) {
+ LOG.warn("Stopping RM while some app masters are still alive");
+ }
+ }
+
@Override
protected synchronized void serviceStop() throws Exception {
if (resourceManager != null) {
+ waitForAppMastersToFinish(5000);
resourceManager.stop();
}
super.serviceStop();
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1535563&r1=1535562&r2=1535563&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Thu Oct 24 21:38:25 2013
@@ -221,10 +221,14 @@ Allocation file format
for containers, but apps submitted later may run concurrently if there is
leftover space on the cluster after satisfying the earlier app's requests.
- * aclSubmitApps: a list of users that can submit apps to the queue. A (default)
- value of "*" means that any users can submit apps. A queue inherits the ACL of
- its parent, so if a queue2 descends from queue1, and user1 is in queue1's ACL,
- and user2 is in queue2's ACL, then both users may submit to queue2.
+ * aclSubmitApps: a list of users and/or groups that can submit apps to the
+ queue. Refer to the ACLs section below for more info on the format of this
+ list and how queue ACLs work.
+
+ * aclAdministerApps: a list of users and/or groups that can administer a
+ queue. Currently the only administrative action is killing an application.
+ Refer to the ACLs section below for more info on the format of this list
+ and how queue ACLs work.
* minSharePreemptionTimeout: number of seconds the queue is under its minimum share
before it will try to preempt containers to take resources from other queues.
@@ -246,6 +250,24 @@ Allocation file format
An example allocation file is given here:
+Queue Access Control Lists (ACLs)
+
+ Queue Access Control Lists (ACLs) allow administrators to control who may
+ take actions on particular queues. They are configured with the aclSubmitApps
+ and aclAdministerApps properties, which can be set per queue. Currently the
+ only supported administrative action is killing an application. Anybody who
+ may administer a queue may also submit applications to it. These properties
+ take values in a format like "user1,user2 group1,group2" or " group1,group2".
+ An action on a queue will be permitted if its user or group is in the ACL of
+ that queue or in the ACL of any of that queue's ancestors. So if queue2
+ is inside queue1, and user1 is in queue1's ACL, and user2 is in queue2's
+ ACL, then both users may submit to queue2.
+
+ The root queue's ACLs are "*" by default which, because ACLs are passed down,
+ means that everybody may submit to and kill applications from every queue.
+ To start restricting access, change the root queue's ACLs to something other
+ than "*".
+
---
<?xml version="1.0"?>
<allocations>
@@ -256,6 +278,7 @@ Allocation file format
<weight>2.0</weight>
<schedulingPolicy>fair</schedulingPolicy>
<queue name="sample_sub_queue">
+ <aclSubmitApps>charlie</aclSubmitApps>
<minResources>5000 mb,0vcores</minResources>
</queue>
</queue>