You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cn...@apache.org on 2014/01/09 00:10:06 UTC
svn commit: r1556665 [1/2] - in
/hadoop/common/branches/HDFS-4685/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-api/src/main/proto/server/
hadoop-yarn/hadoop-yarn-client/ hadoop-y...
Author: cnauroth
Date: Wed Jan 8 23:10:02 2014
New Revision: 1556665
URL: http://svn.apache.org/r1556665
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
- copied unchanged from r1556663, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
- copied unchanged from r1556663, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEvent.java
- copied unchanged from r1556663, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEvent.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
- copied unchanged from r1556663, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMFatalEventType.java
Removed:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreOperationFailedEvent.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreOperationFailedEventType.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMHAServiceTarget.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt Wed Jan 8 23:10:02 2014
@@ -55,6 +55,9 @@ Release 2.4.0 - UNRELEASED
YARN-1028. Added FailoverProxyProvider capability to ResourceManager to help
with RM failover. (Karthik Kambatla via vinodkv)
+ YARN-1029. Added embedded leader election in the ResourceManager. (Karthik
+ Kambatla via vinodkv)
+
IMPROVEMENTS
YARN-7. Support CPU resource for DistributedShell. (Junping Du via llu)
@@ -197,6 +200,11 @@ Release 2.4.0 - UNRELEASED
YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize
app-attempts separately from apps. (Jian He via vinodkv)
+ YARN-1482. Modified WebApplicationProxy to make it work across ResourceManager
+ fail-over. (Xuan Gong via vinodkv)
+
+ YARN-1568. Rename clusterid to clusterId in ActiveRMInfoProto (kasha)
+
OPTIMIZATIONS
BUG FIXES
@@ -292,6 +300,12 @@ Release 2.4.0 - UNRELEASED
YARN-1560. Fixed TestYarnClient#testAMMRTokens failure with null AMRM token.
(Ted Yu via jianhe)
+ YARN-1409. NonAggregatingLogHandler can throw RejectedExecutionException
+ (Tsuyoshi OZAWA via jlowe)
+
+ YARN-1293. Fixed TestContainerLaunch#testInvalidEnvSyntaxDiagnostics failure
+ caused by non-English system locale. (Tsuyoshi OZAWA via jianhe)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Wed Jan 8 23:10:02 2014
@@ -51,6 +51,22 @@ public class HAUtil {
YarnConfiguration.DEFAULT_RM_HA_ENABLED);
}
+ public static boolean isAutomaticFailoverEnabled(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED,
+ YarnConfiguration.DEFAULT_AUTO_FAILOVER_ENABLED);
+ }
+
+ public static boolean isAutomaticFailoverEnabledAndEmbedded(
+ Configuration conf) {
+ return isAutomaticFailoverEnabled(conf) &&
+ isAutomaticFailoverEmbedded(conf);
+ }
+
+ public static boolean isAutomaticFailoverEmbedded(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED,
+ YarnConfiguration.DEFAULT_AUTO_FAILOVER_EMBEDDED);
+ }
+
/**
* Verify configuration for Resource Manager HA.
* @param conf Configuration
@@ -162,8 +178,7 @@ public class HAUtil {
* @param conf Configuration. Please use verifyAndSetRMHAId to check.
* @return RM Id on success
*/
- @VisibleForTesting
- static String getRMHAId(Configuration conf) {
+ public static String getRMHAId(Configuration conf) {
return conf.get(YarnConfiguration.RM_HA_ID);
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Jan 8 23:10:02 2014
@@ -59,7 +59,7 @@ public class YarnConfiguration extends C
public static final String IPC_PREFIX = YARN_PREFIX + "ipc.";
/** Factory to create client IPC classes.*/
- public static final String IPC_CLIENT_FACTORY_CLASS =
+ public static final String IPC_CLIENT_FACTORY_CLASS =
IPC_PREFIX + "client.factory.class";
public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS =
"org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl";
@@ -87,6 +87,8 @@ public class YarnConfiguration extends C
////////////////////////////////
public static final String RM_PREFIX = "yarn.resourcemanager.";
+ public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id";
+
/** The address of the applications manager interface in the RM.*/
public static final String RM_ADDRESS =
RM_PREFIX + "address";
@@ -278,6 +280,36 @@ public class YarnConfiguration extends C
public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled";
public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
+ /** Zookeeper interaction configs */
+ public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-";
+
+ public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address";
+
+ public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries";
+ public static final int DEFAULT_ZK_RM_NUM_RETRIES = 500;
+
+ public static final String RM_ZK_RETRY_INTERVAL_MS =
+ RM_ZK_PREFIX + "retry-interval-ms";
+ public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 2000;
+
+ public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms";
+ public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000;
+
+ public static final String RM_ZK_ACL = RM_ZK_PREFIX + "acl";
+ public static final String DEFAULT_RM_ZK_ACL = "world:anyone:rwcda";
+
+ public static final String ZK_STATE_STORE_PREFIX =
+ RM_PREFIX + "zk-state-store.";
+
+ /** Parent znode path under which ZKRMStateStore will create znodes */
+ public static final String ZK_RM_STATE_STORE_PARENT_PATH =
+ ZK_STATE_STORE_PREFIX + "parent-path";
+ public static final String DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH = "/rmstore";
+
+ /** Root node ACLs for fencing */
+ public static final String ZK_RM_STATE_STORE_ROOT_NODE_ACL =
+ ZK_STATE_STORE_PREFIX + "root-node.acl";
+
/** HA related configs */
public static final String RM_HA_PREFIX = RM_PREFIX + "ha.";
public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled";
@@ -296,6 +328,22 @@ public class YarnConfiguration extends C
HttpConfig.isSecure() ? RM_WEBAPP_HTTPS_ADDRESS
: RM_WEBAPP_ADDRESS));
+ public static final String AUTO_FAILOVER_PREFIX =
+ RM_HA_PREFIX + "automatic-failover.";
+
+ public static final String AUTO_FAILOVER_ENABLED =
+ AUTO_FAILOVER_PREFIX + "enabled";
+ public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = false;
+
+ public static final String AUTO_FAILOVER_EMBEDDED =
+ AUTO_FAILOVER_PREFIX + "embedded";
+ public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = false;
+
+ public static final String AUTO_FAILOVER_ZK_BASE_PATH =
+ AUTO_FAILOVER_PREFIX + "zk-base-path";
+ public static final String DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH =
+ "/yarn-leader-election";
+
public static final String CLIENT_FAILOVER_PREFIX =
YARN_PREFIX + "client.failover-";
public static final String CLIENT_FAILOVER_PROXY_PROVIDER =
@@ -334,36 +382,6 @@ public class YarnConfiguration extends C
+ "fs.state-store.retry-policy-spec";
public static final String DEFAULT_FS_RM_STATE_STORE_RETRY_POLICY_SPEC =
"2000, 500";
- /**
- * Comma separated host:port pairs, each corresponding to a ZK server for
- * ZKRMStateStore
- */
- public static final String ZK_STATE_STORE_PREFIX =
- RM_PREFIX + "zk-state-store.";
- public static final String ZK_RM_STATE_STORE_NUM_RETRIES =
- ZK_STATE_STORE_PREFIX + "num-retries";
- public static final int DEFAULT_ZK_RM_STATE_STORE_NUM_RETRIES = 500;
- /** retry interval when connecting to zookeeper*/
- public static final String ZK_RM_STATE_STORE_RETRY_INTERVAL_MS =
- ZK_STATE_STORE_PREFIX + "retry-interval-ms";
- public static final long DEFAULT_ZK_RM_STATE_STORE_RETRY_INTERVAL_MS = 2000;
- public static final String ZK_RM_STATE_STORE_ADDRESS =
- ZK_STATE_STORE_PREFIX + "address";
- /** Timeout in millisec for ZK server connection for ZKRMStateStore */
- public static final String ZK_RM_STATE_STORE_TIMEOUT_MS =
- ZK_STATE_STORE_PREFIX + "timeout-ms";
- public static final int DEFAULT_ZK_RM_STATE_STORE_TIMEOUT_MS = 60000;
- /** Parent znode path under which ZKRMStateStore will create znodes */
- public static final String ZK_RM_STATE_STORE_PARENT_PATH =
- ZK_STATE_STORE_PREFIX + "parent-path";
- public static final String DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH = "/rmstore";
- /** ACL for znodes in ZKRMStateStore */
- public static final String ZK_RM_STATE_STORE_ACL =
- ZK_STATE_STORE_PREFIX + "acl";
- public static final String DEFAULT_ZK_RM_STATE_STORE_ACL =
- "world:anyone:rwcda";
- public static final String ZK_RM_STATE_STORE_ROOT_NODE_ACL =
- ZK_STATE_STORE_PREFIX + "root-node.acl";
/** The maximum number of completed applications RM keeps. */
public static final String RM_MAX_COMPLETED_APPLICATIONS =
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto Wed Jan 8 23:10:02 2014
@@ -133,3 +133,11 @@ message RMStateVersionProto {
optional int32 major_version = 1;
optional int32 minor_version = 2;
}
+
+//////////////////////////////////////////////////////////////////
+///////////// RM Failover related records ////////////////////////
+//////////////////////////////////////////////////////////////////
+message ActiveRMInfoProto {
+ required string clusterId = 1;
+ required string rmId = 2;
+}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml Wed Jan 8 23:10:02 2014
@@ -30,7 +30,13 @@
</properties>
<dependencies>
- <dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
</dependency>
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java Wed Jan 8 23:10:02 2014
@@ -24,41 +24,52 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
+import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
-public class TestRMFailover {
+public class TestRMFailover extends ClientBaseWithFixes {
private static final Log LOG =
LogFactory.getLog(TestRMFailover.class.getName());
+ private static final HAServiceProtocol.StateChangeRequestInfo req =
+ new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
private static final String RM1_NODE_ID = "rm1";
private static final int RM1_PORT_BASE = 10000;
private static final String RM2_NODE_ID = "rm2";
private static final int RM2_PORT_BASE = 20000;
- private static final HAServiceProtocol.StateChangeRequestInfo req =
- new HAServiceProtocol.StateChangeRequestInfo(
- HAServiceProtocol.RequestSource.REQUEST_BY_USER_FORCED);
- private static Configuration conf;
- private static MiniYARNCluster cluster;
+ private Configuration conf;
+ private MiniYARNCluster cluster;
+ private ApplicationId fakeAppId;
- private static void setConfForRM(String rmId, String prefix, String value) {
+
+ private void setConfForRM(String rmId, String prefix, String value) {
conf.set(HAUtil.addSuffix(prefix, rmId), value);
}
- private static void setRpcAddressForRM(String rmId, int base) {
+ private void setRpcAddressForRM(String rmId, int base) {
setConfForRM(rmId, YarnConfiguration.RM_ADDRESS, "0.0.0.0:" +
(base + YarnConfiguration.DEFAULT_RM_PORT));
setConfForRM(rmId, YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:" +
@@ -73,13 +84,9 @@ public class TestRMFailover {
(base + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT));
}
- private static AdminService getRMAdminService(int index) {
- return
- cluster.getResourceManager(index).getRMContext().getRMAdminService();
- }
-
- @BeforeClass
- public static void setup() throws IOException {
+ @Before
+ public void setup() throws IOException {
+ fakeAppId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
@@ -87,27 +94,22 @@ public class TestRMFailover {
setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);
conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);
+
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1);
- cluster.init(conf);
- cluster.start();
-
- cluster.getResourceManager(0).getRMContext().getRMAdminService()
- .transitionToActive(req);
- assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
}
- @AfterClass
- public static void teardown() {
+ @After
+ public void teardown() {
cluster.stop();
}
private void verifyClientConnection() {
int numRetries = 3;
while(numRetries-- > 0) {
- Configuration conf = new YarnConfiguration(TestRMFailover.conf);
+ Configuration conf = new YarnConfiguration(this.conf);
YarnClient client = YarnClient.createYarnClient();
client.init(conf);
client.start();
@@ -123,31 +125,131 @@ public class TestRMFailover {
fail("Client couldn't connect to the Active RM");
}
+ private void verifyConnections() throws InterruptedException, YarnException {
+ assertTrue("NMs failed to connect to the RM",
+ cluster.waitForNodeManagersToConnect(20000));
+ verifyClientConnection();
+ }
+
+ private AdminService getAdminService(int index) {
+ return cluster.getResourceManager(index).getRMContext().getRMAdminService();
+ }
+
+ private void explicitFailover() throws IOException {
+ int activeRMIndex = cluster.getActiveRMIndex();
+ int newActiveRMIndex = (activeRMIndex + 1) % 2;
+ getAdminService(activeRMIndex).transitionToStandby(req);
+ getAdminService(newActiveRMIndex).transitionToActive(req);
+ assertEquals("Failover failed", newActiveRMIndex, cluster.getActiveRMIndex());
+ }
+
+ private void failover()
+ throws IOException, InterruptedException, YarnException {
+ int activeRMIndex = cluster.getActiveRMIndex();
+ cluster.stopResourceManager(activeRMIndex);
+ assertEquals("Failover failed",
+ (activeRMIndex + 1) % 2, cluster.getActiveRMIndex());
+ cluster.restartResourceManager(activeRMIndex);
+ }
+
@Test
public void testExplicitFailover()
throws YarnException, InterruptedException, IOException {
- assertTrue("NMs failed to connect to the RM",
- cluster.waitForNodeManagersToConnect(5000));
- verifyClientConnection();
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ cluster.init(conf);
+ cluster.start();
+ getAdminService(0).transitionToActive(req);
+ assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
+ verifyConnections();
- // Failover to the second RM
- getRMAdminService(0).transitionToStandby(req);
- getRMAdminService(1).transitionToActive(req);
- assertEquals("Wrong ResourceManager is active",
- HAServiceProtocol.HAServiceState.ACTIVE,
- getRMAdminService(1).getServiceStatus().getState());
- assertTrue("NMs failed to connect to the RM",
- cluster.waitForNodeManagersToConnect(5000));
- verifyClientConnection();
+ explicitFailover();
+ verifyConnections();
- // Failover back to the first RM
- getRMAdminService(1).transitionToStandby(req);
- getRMAdminService(0).transitionToActive(req);
- assertEquals("Wrong ResourceManager is active",
- HAServiceProtocol.HAServiceState.ACTIVE,
- getRMAdminService(0).getServiceStatus().getState());
- assertTrue("NMs failed to connect to the RM",
- cluster.waitForNodeManagersToConnect(5000));
- verifyClientConnection();
+ explicitFailover();
+ verifyConnections();
+ }
+
+ @Test
+ public void testAutomaticFailover()
+ throws YarnException, InterruptedException, IOException {
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
+ conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000);
+
+ cluster.init(conf);
+ cluster.start();
+ assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
+ verifyConnections();
+
+ failover();
+ verifyConnections();
+
+ failover();
+ verifyConnections();
+ }
+
+ @Test
+ public void testWebAppProxyInStandAloneMode() throws YarnException,
+ InterruptedException, IOException {
+ WebAppProxyServer webAppProxyServer = new WebAppProxyServer();
+ try {
+ conf.set(YarnConfiguration.PROXY_ADDRESS, "0.0.0.0:9099");
+ cluster.init(conf);
+ cluster.start();
+ getAdminService(0).transitionToActive(req);
+ assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
+ verifyConnections();
+ webAppProxyServer.init(conf);
+
+ // Start webAppProxyServer
+ Assert.assertEquals(STATE.INITED, webAppProxyServer.getServiceState());
+ webAppProxyServer.start();
+ Assert.assertEquals(STATE.STARTED, webAppProxyServer.getServiceState());
+
+ URL wrongUrl = new URL("http://0.0.0.0:9099/proxy/" + fakeAppId);
+ HttpURLConnection proxyConn = (HttpURLConnection) wrongUrl
+ .openConnection();
+
+ proxyConn.connect();
+ verifyExpectedException(proxyConn.getResponseMessage());
+
+ explicitFailover();
+ verifyConnections();
+ proxyConn.connect();
+ verifyExpectedException(proxyConn.getResponseMessage());
+ } finally {
+ webAppProxyServer.stop();
+ }
+ }
+
+ @Test
+ public void testEmbeddedWebAppProxy() throws YarnException,
+ InterruptedException, IOException {
+ cluster.init(conf);
+ cluster.start();
+ getAdminService(0).transitionToActive(req);
+ assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
+ verifyConnections();
+ URL wrongUrl = new URL("http://0.0.0.0:18088/proxy/" + fakeAppId);
+ HttpURLConnection proxyConn = (HttpURLConnection) wrongUrl
+ .openConnection();
+
+ proxyConn.connect();
+ verifyExpectedException(proxyConn.getResponseMessage());
+
+ explicitFailover();
+ verifyConnections();
+ proxyConn.connect();
+ verifyExpectedException(proxyConn.getResponseMessage());
+ }
+
+ private void verifyExpectedException(String exceptionMessage){
+ assertTrue(exceptionMessage.contains(ApplicationNotFoundException.class
+ .getName()));
+ assertTrue(exceptionMessage
+ .contains("Application with id '" + fakeAppId + "' " +
+ "doesn't exist in RM."));
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMHAServiceTarget.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMHAServiceTarget.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMHAServiceTarget.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMHAServiceTarget.java Wed Jan 8 23:10:02 2014
@@ -21,16 +21,19 @@ package org.apache.hadoop.yarn.client;
import org.apache.hadoop.ha.BadFencingConfigurationException;
import org.apache.hadoop.ha.HAServiceTarget;
import org.apache.hadoop.ha.NodeFencer;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import java.io.IOException;
import java.net.InetSocketAddress;
public class RMHAServiceTarget extends HAServiceTarget {
- private InetSocketAddress haAdminServiceAddress;
+ private final boolean autoFailoverEnabled;
+ private final InetSocketAddress haAdminServiceAddress;
public RMHAServiceTarget(YarnConfiguration conf)
throws IOException {
+ autoFailoverEnabled = HAUtil.isAutomaticFailoverEnabled(conf);
haAdminServiceAddress = conf.getSocketAddr(
YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
@@ -44,19 +47,23 @@ public class RMHAServiceTarget extends H
@Override
public InetSocketAddress getZKFCAddress() {
- // TODO (YARN-1177): Hook up ZKFC information
- return null;
+ // TODO (YARN-1177): ZKFC implementation
+ throw new UnsupportedOperationException("RMHAServiceTarget doesn't have " +
+ "a corresponding ZKFC address");
}
@Override
public NodeFencer getFencer() {
- // TODO (YARN-1026): Hook up fencing implementation
return null;
}
@Override
- public void checkFencingConfigured()
- throws BadFencingConfigurationException {
- // TODO (YARN-1026): Based on fencing implementation
+ public void checkFencingConfigured() throws BadFencingConfigurationException {
+ throw new BadFencingConfigurationException("Fencer not configured");
+ }
+
+ @Override
+ public boolean isAutoFailoverEnabled() {
+ return autoFailoverEnabled;
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Wed Jan 8 23:10:02 2014
@@ -264,13 +264,20 @@
<property>
<description>Enable RM to recover state after starting. If true, then
- yarn.resourcemanager.store.class must be specified</description>
+ yarn.resourcemanager.store.class must be specified. </description>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>false</value>
</property>
<property>
- <description>The class to use as the persistent store.</description>
+ <description>The class to use as the persistent store.
+
+ If org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
+ is used, the store is implicitly fenced; meaning a single ResourceManager
+ is able to use the store at any point in time. More details on this
+ implicit fencing, along with setting up appropriate ACLs is discussed
+ under yarn.resourcemanager.zk-state-store.root-node.acl.
+ </description>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore</value>
</property>
@@ -291,31 +298,24 @@
</property>
<property>
- <description>Host:Port of the ZooKeeper server where RM state will
- be stored. This must be supplied when using
- org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
- as the value for yarn.resourcemanager.store.class. ZKRMStateStore
- is implicitly fenced, meaning a single ResourceManager is
- able to use the store at any point in time. More details on this, along
- with setting up appropriate ACLs is discussed under the description for
- yarn.resourcemanager.zk-state-store.root-node.acl.</description>
- <name>yarn.resourcemanager.zk-state-store.address</name>
+ <description>Host:Port of the ZooKeeper server to be used by the RM. This
+ must be supplied when using the ZooKeeper based implementation of the
+ RM state store and/or embedded automatic failover in a HA setting.
+ </description>
+ <name>yarn.resourcemanager.zk-address</name>
<!--value>127.0.0.1:2181</value-->
</property>
<property>
- <description>Number of times ZKRMStateStore tries to connect to
- ZooKeeper. This may be supplied when using
- org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
- as the value for yarn.resourcemanager.store.class</description>
- <name>yarn.resourcemanager.zk-state-store.num-retries</name>
+ <description>Number of times RM tries to connect to ZooKeeper.</description>
+ <name>yarn.resourcemanager.zk-num-retries</name>
<value>500</value>
</property>
<property>
- <description>Retry interval in milliseconds when ZKRMStateStore tries to
- connect to ZooKeeper.</description>
- <name>yarn.resourcemanager.zk-state-store.retry-interval-ms</name>
+ <description>Retry interval in milliseconds when connecting to ZooKeeper.
+ </description>
+ <name>yarn.resourcemanager.zk-retry-interval-ms</name>
<value>2000</value>
</property>
@@ -333,20 +333,14 @@
is managed by the ZooKeeper cluster itself, not by the client. This value is
used by the cluster to determine when the client's session expires.
Expirations happens when the cluster does not hear from the client within
- the specified session timeout period (i.e. no heartbeat).
- This may be supplied when using
- org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
- as the value for yarn.resourcemanager.store.class</description>
- <name>yarn.resourcemanager.zk-state-store.timeout-ms</name>
+ the specified session timeout period (i.e. no heartbeat).</description>
+ <name>yarn.resourcemanager.zk-timeout-ms</name>
<value>60000</value>
</property>
<property>
- <description>ACL's to be used for ZooKeeper znodes.
- This may be supplied when using
- org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
- as the value for yarn.resourcemanager.store.class</description>
- <name>yarn.resourcemanager.zk-state-store.acl</name>
+ <description>ACL's to be used for ZooKeeper znodes.</description>
+ <name>yarn.resourcemanager.zk-acl</name>
<value>world:anyone:rwcda</value>
</property>
@@ -362,7 +356,7 @@
permissions.
By default, when this property is not set, we use the ACLs from
- yarn.resourcemanager.zk-state-store.acl for shared admin access and
+ yarn.resourcemanager.zk-acl for shared admin access and
rm-address:cluster-timestamp for username-based exclusive create-delete
access.
@@ -409,6 +403,36 @@
</property>
<property>
+ <description>Enable automatic failover.</description>
+ <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>Enable embedded automatic failover. The embedded elector
+ relies on the RM state store to handle fencing, and is primarily intended
+ to be used in conjunction with ZKRMStateStore.</description>
+ <name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>The base znode path to use for storing leader information,
+ when using ZooKeeper based leader election.</description>
+ <name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name>
+ <value>/yarn-leader-election</value>
+ </property>
+
+ <property>
+ <description>Name of the cluster. In a HA setting,
+ this is used to ensure the RM participates in leader
+ election fo this cluster and ensures it does not affect
+ other clusters</description>
+ <name>yarn.resourcemanager.cluster-id</name>
+ <!--value>yarn-cluster</value-->
+ </property>
+
+ <property>
<description>The list of RM nodes in the cluster when HA is
enabled. See description of yarn.resourcemanager.ha
.enabled for full details on how this is used.</description>
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java Wed Jan 8 23:10:02 2014
@@ -23,6 +23,7 @@ import java.util.concurrent.ConcurrentHa
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.RejectedExecutionException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -118,10 +119,17 @@ public class NonAggregatingLogHandler ex
LOG.info("Scheduling Log Deletion for application: "
+ appFinishedEvent.getApplicationId() + ", with delay of "
+ this.deleteDelaySeconds + " seconds");
- sched.schedule(
+ LogDeleterRunnable logDeleter =
new LogDeleterRunnable(appOwners.remove(appFinishedEvent
- .getApplicationId()), appFinishedEvent.getApplicationId()),
- this.deleteDelaySeconds, TimeUnit.SECONDS);
+ .getApplicationId()), appFinishedEvent.getApplicationId());
+ try {
+ sched.schedule(logDeleter, this.deleteDelaySeconds,
+ TimeUnit.SECONDS);
+ } catch (RejectedExecutionException e) {
+ // Handling this event in local thread before starting threads
+ // or after calling sched.shutdownNow().
+ logDeleter.run();
+ }
break;
default:
; // Ignore
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Wed Jan 8 23:10:02 2014
@@ -260,8 +260,12 @@ public class TestContainerLaunch extends
fos.flush();
fos.close();
+ // It is supposed that LANG is set as C.
+ Map<String, String> cmdEnv = new HashMap<String, String>();
+ cmdEnv.put("LANG", "C");
Shell.ShellCommandExecutor shexc
- = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()}, tmpDir);
+ = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
+ tmpDir, cmdEnv);
String diagnostics = null;
try {
shexc.execute();
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java Wed Jan 8 23:10:02 2014
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.event.InlineDispatcher;
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@@ -202,6 +203,32 @@ public class TestNonAggregatingLogHandle
verify(logHandler.mockSched).shutdownNow();
}
+ @Test
+ public void testHandlingApplicationFinishedEvent() {
+ Configuration conf = new Configuration();
+ LocalDirsHandlerService dirsService = new LocalDirsHandlerService();
+ DeletionService delService = new DeletionService(null);
+ NonAggregatingLogHandler aggregatingLogHandler =
+ new NonAggregatingLogHandler(new InlineDispatcher(),
+ delService,
+ dirsService);
+
+ dirsService.init(conf);
+ dirsService.start();
+ delService.init(conf);
+ delService.start();
+ aggregatingLogHandler.init(conf);
+ aggregatingLogHandler.start();
+ ApplicationId appId = BuilderUtils.newApplicationId(1234, 1);
+ // It should NOT throw RejectedExecutionException
+ aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
+ aggregatingLogHandler.stop();
+
+ // It should NOT throw RejectedExecutionException after stopping
+ // handler service.
+ aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
+ }
+
private class NonAggregatingLogHandlerWithMockExecutor extends
NonAggregatingLogHandler {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Wed Jan 8 23:10:02 2014
@@ -20,11 +20,8 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -41,14 +38,16 @@ import org.apache.hadoop.ipc.ProtobufRpc
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RPC.Server;
import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
-import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -75,7 +74,7 @@ import org.apache.hadoop.yarn.server.res
import com.google.protobuf.BlockingService;
-public class AdminService extends AbstractService implements
+public class AdminService extends CompositeService implements
HAServiceProtocol, ResourceManagerAdministrationProtocol {
private static final Log LOG = LogFactory.getLog(AdminService.class);
@@ -84,6 +83,8 @@ public class AdminService extends Abstra
private final ResourceManager rm;
private String rmId;
+ private boolean autoFailoverEnabled;
+
private Server server;
private InetSocketAddress masterServiceAddress;
private AccessControlList adminAcl;
@@ -99,6 +100,15 @@ public class AdminService extends Abstra
@Override
public synchronized void serviceInit(Configuration conf) throws Exception {
+ if (rmContext.isHAEnabled()) {
+ autoFailoverEnabled = HAUtil.isAutomaticFailoverEnabled(conf);
+ if (autoFailoverEnabled) {
+ if (HAUtil.isAutomaticFailoverEmbedded(conf)) {
+ addIfService(createEmbeddedElectorService());
+ }
+ }
+ }
+
masterServiceAddress = conf.getSocketAddr(
YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
@@ -162,6 +172,10 @@ public class AdminService extends Abstra
}
}
+ protected EmbeddedElectorService createEmbeddedElectorService() {
+ return new EmbeddedElectorService(rmContext);
+ }
+
private UserGroupInformation checkAccess(String method) throws IOException {
return RMServerUtils.verifyAccess(adminAcl, method, LOG);
}
@@ -174,6 +188,43 @@ public class AdminService extends Abstra
}
}
+ /**
+ * Check that a request to change this node's HA state is valid.
+ * In particular, verifies that, if auto failover is enabled, non-forced
+ * requests from the HAAdmin CLI are rejected, and vice versa.
+ *
+ * @param req the request to check
+ * @throws AccessControlException if the request is disallowed
+ */
+ private void checkHaStateChange(StateChangeRequestInfo req)
+ throws AccessControlException {
+ switch (req.getSource()) {
+ case REQUEST_BY_USER:
+ if (autoFailoverEnabled) {
+ throw new AccessControlException(
+ "Manual failover for this ResourceManager is disallowed, " +
+ "because automatic failover is enabled.");
+ }
+ break;
+ case REQUEST_BY_USER_FORCED:
+ if (autoFailoverEnabled) {
+ LOG.warn("Allowing manual failover from " +
+ org.apache.hadoop.ipc.Server.getRemoteAddress() +
+ " even though automatic failover is enabled, because the user " +
+ "specified the force flag");
+ }
+ break;
+ case REQUEST_BY_ZKFC:
+ if (!autoFailoverEnabled) {
+ throw new AccessControlException(
+ "Request from ZK failover controller at " +
+ org.apache.hadoop.ipc.Server.getRemoteAddress() + " denied " +
+ "since automatic failover is not enabled");
+ }
+ break;
+ }
+ }
+
private synchronized boolean isRMActive() {
return HAServiceState.ACTIVE == rmContext.getHAServiceState();
}
@@ -196,8 +247,7 @@ public class AdminService extends Abstra
public synchronized void transitionToActive(
HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException {
UserGroupInformation user = checkAccess("transitionToActive");
- // TODO (YARN-1177): When automatic failover is enabled,
- // check if transition should be allowed for this request
+ checkHaStateChange(reqInfo);
try {
rm.transitionToActive();
RMAuditLogger.logSuccess(user.getShortUserName(),
@@ -215,8 +265,7 @@ public class AdminService extends Abstra
public synchronized void transitionToStandby(
HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException {
UserGroupInformation user = checkAccess("transitionToStandby");
- // TODO (YARN-1177): When automatic failover is enabled,
- // check if transition should be allowed for this request
+ checkHaStateChange(reqInfo);
try {
rm.transitionToStandby(true);
RMAuditLogger.logSuccess(user.getShortUserName(),
@@ -394,7 +443,7 @@ public class AdminService extends Abstra
public String[] getGroupsForUser(String user) throws IOException {
return UserGroupInformation.createRemoteUser(user).getGroupNames();
}
-
+
@Override
public UpdateNodeResourceResponse updateNodeResource(
UpdateNodeResourceRequest request) throws YarnException, IOException {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Wed Jan 8 23:10:02 2014
@@ -59,8 +59,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreFactory;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreOperationFailedEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStoreOperationFailedEventType;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
@@ -121,6 +119,7 @@ public class ResourceManager extends Com
*/
@VisibleForTesting
protected RMContextImpl rmContext;
+ private Dispatcher rmDispatcher;
@VisibleForTesting
protected AdminService adminService;
@@ -134,7 +133,6 @@ public class ResourceManager extends Com
*/
protected RMActiveServices activeServices;
protected RMSecretManagerService rmSecretManagerService;
- private Dispatcher rmDispatcher;
protected ResourceScheduler scheduler;
private ClientRMService clientRM;
@@ -149,9 +147,12 @@ public class ResourceManager extends Com
protected QueueACLsManager queueACLsManager;
private DelegationTokenRenewer delegationTokenRenewer;
private WebApp webApp;
+ private AppReportFetcher fetcher = null;
protected ResourceTrackerService resourceTracker;
private boolean recoveryEnabled;
+ private String webAppAddress;
+
/** End of Active services */
private Configuration conf;
@@ -179,6 +180,13 @@ public class ResourceManager extends Com
this.conf = conf;
this.rmContext = new RMContextImpl();
+ rmDispatcher = createDispatcher();
+ addIfService(rmDispatcher);
+ rmContext.setDispatcher(rmDispatcher);
+
+ rmDispatcher.register(RMFatalEventType.class,
+ new ResourceManager.RMFatalEventDispatcher(this.rmContext, this));
+
adminService = createAdminService();
addService(adminService);
rmContext.setRMAdminService(adminService);
@@ -189,6 +197,8 @@ public class ResourceManager extends Com
}
createAndInitActiveServices();
+ webAppAddress = WebAppUtils.getRMWebAppURLWithoutScheme(conf);
+
super.serviceInit(conf);
}
@@ -207,11 +217,6 @@ public class ResourceManager extends Com
return new SchedulerEventDispatcher(this.scheduler);
}
- protected RMStateStoreOperationFailedEventDispatcher
- createRMStateStoreOperationFailedEventDispatcher() {
- return new RMStateStoreOperationFailedEventDispatcher(rmContext, this);
- }
-
protected Dispatcher createDispatcher() {
return new AsyncDispatcher();
}
@@ -297,10 +302,6 @@ public class ResourceManager extends Com
protected void serviceInit(Configuration configuration) throws Exception {
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
- rmDispatcher = createDispatcher();
- addIfService(rmDispatcher);
- rmContext.setDispatcher(rmDispatcher);
-
rmSecretManagerService = createRMSecretManagerService();
addService(rmSecretManagerService);
@@ -332,8 +333,6 @@ public class ResourceManager extends Com
try {
rmStore.init(conf);
rmStore.setRMDispatcher(rmDispatcher);
- rmDispatcher.register(RMStateStoreOperationFailedEventType.class,
- createRMStateStoreOperationFailedEventDispatcher());
} catch (Exception e) {
// the Exception from stateStore.init() needs to be handled for
// HA and we need to give up master status if we got fenced
@@ -443,22 +442,12 @@ public class ResourceManager extends Com
throw e;
}
}
- startWepApp();
-
- if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- int port = webApp.port();
- WebAppUtils.setRMWebAppPort(conf, port);
- }
super.serviceStart();
}
@Override
protected void serviceStop() throws Exception {
- if (webApp != null) {
- webApp.stop();
- }
-
DefaultMetricsSystem.shutdown();
@@ -605,26 +594,23 @@ public class ResourceManager extends Com
}
@Private
- public static class RMStateStoreOperationFailedEventDispatcher implements
- EventHandler<RMStateStoreOperationFailedEvent> {
-
+ public static class RMFatalEventDispatcher
+ implements EventHandler<RMFatalEvent> {
private final RMContext rmContext;
private final ResourceManager rm;
- public RMStateStoreOperationFailedEventDispatcher(RMContext rmContext,
- ResourceManager resourceManager) {
+ public RMFatalEventDispatcher(
+ RMContext rmContext, ResourceManager resourceManager) {
this.rmContext = rmContext;
this.rm = resourceManager;
}
@Override
- public void handle(RMStateStoreOperationFailedEvent event) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received a " +
- RMStateStoreOperationFailedEvent.class.getName() + " of type " +
- event.getType().name());
- }
- if (event.getType() == RMStateStoreOperationFailedEventType.FENCED) {
+ public void handle(RMFatalEvent event) {
+ LOG.fatal("Received a " + RMFatalEvent.class.getName() + " of type " +
+ event.getType().name());
+
+ if (event.getType() == RMFatalEventType.STATE_STORE_FENCED) {
LOG.info("RMStateStore has been fenced");
if (rmContext.isHAEnabled()) {
try {
@@ -633,14 +619,11 @@ public class ResourceManager extends Com
rm.transitionToStandby(true);
return;
} catch (Exception e) {
- LOG.error("Failed to transition RM to Standby mode.");
+ LOG.fatal("Failed to transition RM to Standby mode.");
}
}
}
- LOG.error("Shutting down RM on receiving a " +
- RMStateStoreOperationFailedEvent.class.getName() + " of type " +
- event.getType().name());
ExitUtil.terminate(1, event.getCause());
}
}
@@ -764,12 +747,16 @@ public class ResourceManager extends Com
YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY)
.withHttpSpnegoKeytabKey(
YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
- .at(WebAppUtils.getRMWebAppURLWithoutScheme(conf));
+ .at(webAppAddress);
String proxyHostAndPort = WebAppUtils.getProxyHostAndPort(conf);
if(WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf).
equals(proxyHostAndPort)) {
- AppReportFetcher fetcher = new AppReportFetcher(conf, getClientRMService());
- builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
+ if (HAUtil.isHAEnabled(conf)) {
+ fetcher = new AppReportFetcher(conf);
+ } else {
+ fetcher = new AppReportFetcher(conf, getClientRMService());
+ }
+ builder.withServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
builder.withAttribute(WebAppProxy.FETCHER_ATTRIBUTE, fetcher);
String[] proxyParts = proxyHostAndPort.split(":");
@@ -866,6 +853,11 @@ public class ResourceManager extends Com
transitionToActive();
}
+ startWepApp();
+ if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+ int port = webApp.port();
+ WebAppUtils.setRMWebAppPort(conf, port);
+ }
super.serviceStart();
}
@@ -876,6 +868,12 @@ public class ResourceManager extends Com
@Override
protected void serviceStop() throws Exception {
+ if (webApp != null) {
+ webApp.stop();
+ }
+ if (fetcher != null) {
+ fetcher.stop();
+ }
super.serviceStop();
transitionToStandby(false);
rmContext.setHAServiceState(HAServiceState.STOPPING);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Wed Jan 8 23:10:02 2014
@@ -48,6 +48,8 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEventType;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@@ -679,15 +681,13 @@ public abstract class RMStateStore exten
* @param failureCause the exception due to which the operation failed
*/
private void notifyStoreOperationFailed(Exception failureCause) {
- RMStateStoreOperationFailedEventType type;
+ RMFatalEventType type;
if (failureCause instanceof StoreFencedException) {
- type = RMStateStoreOperationFailedEventType.FENCED;
+ type = RMFatalEventType.STATE_STORE_FENCED;
} else {
- type = RMStateStoreOperationFailedEventType.FAILED;
+ type = RMFatalEventType.STATE_STORE_OP_FAILED;
}
-
- rmDispatcher.getEventHandler().handle(
- new RMStateStoreOperationFailedEvent(type, failureCause));
+ rmDispatcher.getEventHandler().handle(new RMFatalEvent(type, failureCause));
}
@SuppressWarnings("unchecked")
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Wed Jan 8 23:10:02 2014
@@ -182,34 +182,34 @@ public class ZKRMStateStore extends RMSt
@Override
public synchronized void initInternal(Configuration conf) throws Exception {
- zkHostPort = conf.get(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS);
+ zkHostPort = conf.get(YarnConfiguration.RM_ZK_ADDRESS);
if (zkHostPort == null) {
throw new YarnRuntimeException("No server address specified for " +
"zookeeper state store for Resource Manager recovery. " +
- YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS + " is not configured.");
+ YarnConfiguration.RM_ZK_ADDRESS + " is not configured.");
}
numRetries =
- conf.getInt(YarnConfiguration.ZK_RM_STATE_STORE_NUM_RETRIES,
- YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_NUM_RETRIES);
+ conf.getInt(YarnConfiguration.RM_ZK_NUM_RETRIES,
+ YarnConfiguration.DEFAULT_ZK_RM_NUM_RETRIES);
znodeWorkingPath =
conf.get(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH,
YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH);
zkSessionTimeout =
- conf.getInt(YarnConfiguration.ZK_RM_STATE_STORE_TIMEOUT_MS,
- YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_TIMEOUT_MS);
+ conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
+ YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
zkRetryInterval =
- conf.getLong(YarnConfiguration.ZK_RM_STATE_STORE_RETRY_INTERVAL_MS,
- YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_RETRY_INTERVAL_MS);
+ conf.getLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS);
// Parse authentication from configuration.
String zkAclConf =
- conf.get(YarnConfiguration.ZK_RM_STATE_STORE_ACL,
- YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_ACL);
+ conf.get(YarnConfiguration.RM_ZK_ACL,
+ YarnConfiguration.DEFAULT_RM_ZK_ACL);
zkAclConf = ZKUtil.resolveConfIndirection(zkAclConf);
try {
zkAcl = ZKUtil.parseACLs(zkAclConf);
} catch (ZKUtil.BadAclFormatException bafe) {
- LOG.error("Invalid format for " + YarnConfiguration.ZK_RM_STATE_STORE_ACL);
+ LOG.error("Invalid format for " + YarnConfiguration.RM_ZK_ACL);
throw bafe;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Wed Jan 8 23:10:02 2014
@@ -396,6 +396,11 @@ public class MockRM extends ResourceMana
protected void stopServer() {
// don't do anything
}
+
+ @Override
+ protected EmbeddedElectorService createEmbeddedElectorService() {
+ return null;
+ }
};
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Wed Jan 8 23:10:02 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HealthCheckFailedException;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.junit.Before;
@@ -39,6 +40,7 @@ import static org.junit.Assert.fail;
public class TestRMHA {
private Log LOG = LogFactory.getLog(TestRMHA.class);
+ private final Configuration configuration = new YarnConfiguration();
private MockRM rm = null;
private static final String STATE_ERR =
"ResourceManager is in wrong HA state";
@@ -51,17 +53,13 @@ public class TestRMHA {
@Before
public void setUp() throws Exception {
- Configuration conf = new YarnConfiguration();
- conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
- conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
+ configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
- conf.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
- conf.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
+ configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
+ configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
}
- conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
-
- rm = new MockRM(conf);
- rm.init(conf);
+ configuration.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
}
private void checkMonitorHealth() throws IOException {
@@ -113,6 +111,9 @@ public class TestRMHA {
*/
@Test (timeout = 30000)
public void testStartAndTransitions() throws IOException {
+ Configuration conf = new YarnConfiguration(configuration);
+ rm = new MockRM(conf);
+ rm.init(conf);
StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
HAServiceProtocol.RequestSource.REQUEST_BY_USER);
@@ -162,4 +163,63 @@ public class TestRMHA {
rm.areActiveServicesRunning());
checkMonitorHealth();
}
+
+ @Test
+ public void testTransitionsWhenAutomaticFailoverEnabled() throws IOException {
+ final String ERR_UNFORCED_REQUEST = "User request succeeded even when " +
+ "automatic failover is enabled";
+
+ Configuration conf = new YarnConfiguration(configuration);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
+
+ rm = new MockRM(conf);
+ rm.init(conf);
+ rm.start();
+ StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+
+ // Transition to standby
+ try {
+ rm.adminService.transitionToStandby(requestInfo);
+ fail(ERR_UNFORCED_REQUEST);
+ } catch (AccessControlException e) {
+ // expected
+ }
+ checkMonitorHealth();
+ checkStandbyRMFunctionality();
+
+ // Transition to active
+ try {
+ rm.adminService.transitionToActive(requestInfo);
+ fail(ERR_UNFORCED_REQUEST);
+ } catch (AccessControlException e) {
+ // expected
+ }
+ checkMonitorHealth();
+ checkStandbyRMFunctionality();
+
+
+ final String ERR_FORCED_REQUEST = "Forced request by user should work " +
+ "even if automatic failover is enabled";
+ requestInfo = new StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER_FORCED);
+
+ // Transition to standby
+ try {
+ rm.adminService.transitionToStandby(requestInfo);
+ } catch (AccessControlException e) {
+ fail(ERR_FORCED_REQUEST);
+ }
+ checkMonitorHealth();
+ checkStandbyRMFunctionality();
+
+ // Transition to active
+ try {
+ rm.adminService.transitionToActive(requestInfo);
+ } catch (AccessControlException e) {
+ fail(ERR_FORCED_REQUEST);
+ }
+ checkMonitorHealth();
+ checkActiveRMFunctionality();
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Wed Jan 8 23:10:02 2014
@@ -92,7 +92,7 @@ public class TestZKRMStateStore extends
public RMStateStore getRMStateStore() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
workingZnode = "/Test";
- conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS, hostPort);
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
this.client = createClient();
this.store = new TestZKRMStateStoreInternal(conf, workingZnode);
@@ -140,7 +140,7 @@ public class TestZKRMStateStore extends
conf.set(YarnConfiguration.RM_HA_IDS, rmIds);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName());
- conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS, hostPort);
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
conf.set(YarnConfiguration.RM_HA_ID, rmId);
for (String rpcAddress : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
for (String id : HAUtil.getRMHAIds(conf)) {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java?rev=1556665&r1=1556664&r2=1556665&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java Wed Jan 8 23:10:02 2014
@@ -107,7 +107,7 @@ public class TestZKRMStateStoreZKClientC
public RMStateStore getRMStateStore(Configuration conf) throws Exception {
String workingZnode = "/Test";
- conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS, hostPort);
+ conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
watcher = new TestForwardingWatcher();
this.store = new TestZKRMStateStore(conf, workingZnode);
@@ -120,8 +120,8 @@ public class TestZKRMStateStoreZKClientC
TestZKClient zkClientTester = new TestZKClient();
final String path = "/test";
YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.ZK_RM_STATE_STORE_TIMEOUT_MS, 1000);
- conf.setLong(YarnConfiguration.ZK_RM_STATE_STORE_RETRY_INTERVAL_MS, 100);
+ conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 1000);
+ conf.setLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS, 100);
final ZKRMStateStore store =
(ZKRMStateStore) zkClientTester.getRMStateStore(conf);
TestDispatcher dispatcher = new TestDispatcher();
@@ -153,7 +153,7 @@ public class TestZKRMStateStoreZKClientC
TestZKClient zkClientTester = new TestZKClient();
String path = "/test";
YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.ZK_RM_STATE_STORE_TIMEOUT_MS, 100);
+ conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 100);
ZKRMStateStore store =
(ZKRMStateStore) zkClientTester.getRMStateStore(conf);
TestDispatcher dispatcher = new TestDispatcher();
@@ -195,7 +195,7 @@ public class TestZKRMStateStoreZKClientC
TestZKClient zkClientTester = new TestZKClient();
String path = "/test";
YarnConfiguration conf = new YarnConfiguration();
- conf.setInt(YarnConfiguration.ZK_RM_STATE_STORE_TIMEOUT_MS, 100);
+ conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 100);
ZKRMStateStore store =
(ZKRMStateStore) zkClientTester.getRMStateStore(conf);
TestDispatcher dispatcher = new TestDispatcher();
@@ -227,7 +227,7 @@ public class TestZKRMStateStoreZKClientC
public void testSetZKAcl() {
TestZKClient zkClientTester = new TestZKClient();
YarnConfiguration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ACL, "world:anyone:rwca");
+ conf.set(YarnConfiguration.RM_ZK_ACL, "world:anyone:rwca");
try {
zkClientTester.store.zkClient.delete(zkClientTester.store
.znodeWorkingPath, -1);
@@ -240,7 +240,7 @@ public class TestZKRMStateStoreZKClientC
public void testInvalidZKAclConfiguration() {
TestZKClient zkClientTester = new TestZKClient();
YarnConfiguration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ACL, "randomstring&*");
+ conf.set(YarnConfiguration.RM_ZK_ACL, "randomstring&*");
try {
zkClientTester.getRMStateStore(conf);
fail("ZKRMStateStore created with bad ACL");