You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/06/08 19:19:14 UTC
[16/50] hadoop git commit: YARN-3749. We should make a copy of
configuration when init MiniYARNCluster with multiple RMs. Contributed by
Chun Chen
YARN-3749. We should make a copy of configuration when init
MiniYARNCluster with multiple RMs. Contributed by Chun Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d39b344
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d39b344
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d39b344
Branch: refs/heads/YARN-2928
Commit: 8d39b344afe459309d73fe647e3459addf0415a4
Parents: 2e58586
Author: Xuan <xg...@apache.org>
Authored: Wed Jun 3 17:20:15 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Jun 8 09:43:15 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 2 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 26 ++------------
...estApplicationMasterServiceProtocolOnHA.java | 10 +++---
.../hadoop/yarn/client/TestRMFailover.java | 26 ++------------
.../hadoop/yarn/conf/TestYarnConfiguration.java | 22 ++++++++++++
.../ApplicationMasterService.java | 19 +++++-----
.../yarn/server/resourcemanager/HATestUtil.java | 38 ++++++++++++++++++++
.../resourcemanager/TestRMEmbeddedElector.java | 24 ++-----------
.../hadoop/yarn/server/MiniYARNCluster.java | 22 +++++++-----
.../hadoop/yarn/server/TestMiniYarnCluster.java | 37 +++++++++++++++++++
11 files changed, 138 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 95a2325..61cc501 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -572,6 +572,9 @@ Release 2.8.0 - UNRELEASED
YARN-3762. FairScheduler: CME on FSParentQueue#getQueueUserAclInfo. (kasha)
+ YARN-3749. We should make a copy of configuration when init MiniYARNCluster
+ with multiple RMs. (Chun Chen via xgong)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ba07c80..e4ae2b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1991,7 +1991,7 @@ public class YarnConfiguration extends Configuration {
public InetSocketAddress updateConnectAddr(String name,
InetSocketAddress addr) {
String prefix = name;
- if (HAUtil.isHAEnabled(this)) {
+ if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) {
prefix = HAUtil.addSuffix(prefix, HAUtil.getRMHAId(this));
}
return super.updateConnectAddr(prefix, addr);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 903dd94..75e6cee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
@@ -101,7 +102,6 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
@@ -161,26 +161,6 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
protected Thread failoverThread = null;
private volatile boolean keepRunning;
- private void setConfForRM(String rmId, String prefix, String value) {
- conf.set(HAUtil.addSuffix(prefix, rmId), value);
- }
-
- private void setRpcAddressForRM(String rmId, int base) {
- setConfForRM(rmId, YarnConfiguration.RM_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_ADMIN_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_ADMIN_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
- "0.0.0.0:" + (base + YarnConfiguration
- .DEFAULT_RM_RESOURCE_TRACKER_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_WEBAPP_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT));
- }
-
@Before
public void setup() throws IOException {
failoverThread = null;
@@ -189,8 +169,8 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.setInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 5);
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
- setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
- setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);
+ HATestUtil.setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE, conf);
+ HATestUtil.setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE, conf);
conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java
index 41e1800..ad86fb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceProtocolOnHA.java
@@ -54,18 +54,18 @@ public class TestApplicationMasterServiceProtocolOnHA
public void initialize() throws Exception {
startHACluster(0, false, false, true);
attemptId = this.cluster.createFakeApplicationAttemptId();
- amClient = ClientRMProxy
- .createRMProxy(this.conf, ApplicationMasterProtocol.class);
Token<AMRMTokenIdentifier> appToken =
this.cluster.getResourceManager().getRMContext()
.getAMRMTokenSecretManager().createAndGetAMRMToken(attemptId);
- appToken.setService(ClientRMProxy.getAMRMTokenService(conf));
+ appToken.setService(ClientRMProxy.getAMRMTokenService(this.conf));
UserGroupInformation.setLoginUser(UserGroupInformation
- .createRemoteUser(UserGroupInformation.getCurrentUser()
- .getUserName()));
+ .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
UserGroupInformation.getCurrentUser().addToken(appToken);
syncToken(appToken);
+
+ amClient = ClientRMProxy
+ .createRMProxy(this.conf, ApplicationMasterProtocol.class);
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index cd22743..0d03fd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -38,11 +38,11 @@ import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
+import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.junit.After;
@@ -66,34 +66,14 @@ public class TestRMFailover extends ClientBaseWithFixes {
private MiniYARNCluster cluster;
private ApplicationId fakeAppId;
-
- private void setConfForRM(String rmId, String prefix, String value) {
- conf.set(HAUtil.addSuffix(prefix, rmId), value);
- }
-
- private void setRpcAddressForRM(String rmId, int base) {
- setConfForRM(rmId, YarnConfiguration.RM_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_ADMIN_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_ADMIN_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_WEBAPP_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT));
- }
-
@Before
public void setup() throws IOException {
fakeAppId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
- setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
- setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);
+ HATestUtil.setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE, conf);
+ HATestUtil.setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE, conf);
conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
index 1d925a7..1f10810 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
@@ -27,6 +27,7 @@ import java.net.InetSocketAddress;
import java.net.SocketAddress;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
@@ -202,5 +203,26 @@ public class TestYarnConfiguration {
serverAddress);
assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
+
+ //tests updateConnectAddr won't add suffix to NM service address configurations
+ conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "yo.yo.yo");
+ conf.set(YarnConfiguration.NM_BIND_HOST, "0.0.0.0");
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_ID, "rm1");
+
+ serverAddress = new InetSocketAddress(
+ YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[0],
+ Integer.valueOf(YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS.split(":")[1]));
+
+ InetSocketAddress localizerAddress = conf.updateConnectAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_LOCALIZER_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
+ serverAddress);
+
+ assertTrue(localizerAddress.toString().startsWith("yo.yo.yo"));
+ assertNull(conf.get(
+ HAUtil.addSuffix(YarnConfiguration.NM_LOCALIZER_ADDRESS, "rm1")));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index f5474bf..3637f91 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -107,7 +107,7 @@ public class ApplicationMasterService extends AbstractService implements
private static final Log LOG = LogFactory.getLog(ApplicationMasterService.class);
private final AMLivelinessMonitor amLivelinessMonitor;
private YarnScheduler rScheduler;
- private InetSocketAddress bindAddress;
+ private InetSocketAddress masterServiceAddress;
private Server server;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -123,15 +123,18 @@ public class ApplicationMasterService extends AbstractService implements
}
@Override
- protected void serviceStart() throws Exception {
- Configuration conf = getConfig();
- YarnRPC rpc = YarnRPC.create(conf);
-
- InetSocketAddress masterServiceAddress = conf.getSocketAddr(
+ protected void serviceInit(Configuration conf) throws Exception {
+ masterServiceAddress = conf.getSocketAddr(
YarnConfiguration.RM_BIND_HOST,
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ Configuration conf = getConfig();
+ YarnRPC rpc = YarnRPC.create(conf);
Configuration serverConf = conf;
// If the auth is not-simple, enforce it to be token-based.
@@ -160,7 +163,7 @@ public class ApplicationMasterService extends AbstractService implements
}
this.server.start();
- this.bindAddress =
+ this.masterServiceAddress =
conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST,
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
@@ -170,7 +173,7 @@ public class ApplicationMasterService extends AbstractService implements
@Private
public InetSocketAddress getBindAddress() {
- return this.bindAddress;
+ return this.masterServiceAddress;
}
// Obtain the needed AMRMTokenIdentifier from the remote-UGI. RPC layer
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/HATestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/HATestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/HATestUtil.java
new file mode 100644
index 0000000..710ce87
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/HATestUtil.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+public class HATestUtil {
+
+ public static void setRpcAddressForRM(String rmId, int base,
+ Configuration conf) {
+ for (String confKey : YarnConfiguration.getServiceAddressConfKeys(conf)) {
+ setConfForRM(rmId, confKey, "0.0.0.0:" + (base +
+ YarnConfiguration.getRMDefaultPortNumber(confKey, conf)), conf);
+ }
+ }
+
+ public static void setConfForRM(String rmId, String prefix, String value,
+ Configuration conf) {
+ conf.set(HAUtil.addSuffix(prefix, rmId), value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
index 0d9ee6d..1b0bf7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
@@ -22,7 +22,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.ha.ServiceFailedException;
-import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Before;
import org.junit.Test;
@@ -42,25 +41,6 @@ public class TestRMEmbeddedElector extends ClientBaseWithFixes {
private Configuration conf;
private AtomicBoolean callbackCalled;
- private void setConfForRM(String rmId, String prefix, String value) {
- conf.set(HAUtil.addSuffix(prefix, rmId), value);
- }
-
- private void setRpcAddressForRM(String rmId, int base) {
- setConfForRM(rmId, YarnConfiguration.RM_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_SCHEDULER_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_ADMIN_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_ADMIN_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_WEBAPP_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_WEBAPP_PORT));
- setConfForRM(rmId, YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, "0.0.0.0:" +
- (base + YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT));
- }
-
@Before
public void setup() throws IOException {
conf = new YarnConfiguration();
@@ -73,8 +53,8 @@ public class TestRMEmbeddedElector extends ClientBaseWithFixes {
conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
conf.set(YarnConfiguration.RM_HA_ID, RM1_NODE_ID);
- setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
- setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);
+ HATestUtil.setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE, conf);
+ HATestUtil.setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE, conf);
conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 65c2760..7fb0698 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -287,10 +287,12 @@ public class MiniYARNCluster extends CompositeService {
}
private synchronized void initResourceManager(int index, Configuration conf) {
- if (HAUtil.isHAEnabled(conf)) {
- conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
+ Configuration newConf = resourceManagers.length > 1 ?
+ new YarnConfiguration(conf) : conf;
+ if (HAUtil.isHAEnabled(newConf)) {
+ newConf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
}
- resourceManagers[index].init(conf);
+ resourceManagers[index].init(newConf);
resourceManagers[index].getRMContext().getDispatcher().register(
RMAppAttemptEventType.class,
new EventHandler<RMAppAttemptEvent>() {
@@ -329,10 +331,11 @@ public class MiniYARNCluster extends CompositeService {
} catch (Throwable t) {
throw new YarnRuntimeException(t);
}
+ Configuration conf = resourceManagers[index].getConfig();
LOG.info("MiniYARN ResourceManager address: " +
- getConfig().get(YarnConfiguration.RM_ADDRESS));
+ conf.get(YarnConfiguration.RM_ADDRESS));
LOG.info("MiniYARN ResourceManager web address: " +
- WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
+ WebAppUtils.getRMWebAppURLWithoutScheme(conf));
}
@InterfaceAudience.Private
@@ -352,7 +355,6 @@ public class MiniYARNCluster extends CompositeService {
resourceManagers[index].stop();
resourceManagers[index] = null;
}
- Configuration conf = getConfig();
resourceManagers[index] = new ResourceManager();
initResourceManager(index, getConfig());
startResourceManager(index);
@@ -433,6 +435,7 @@ public class MiniYARNCluster extends CompositeService {
private class ResourceManagerWrapper extends AbstractService {
private int index;
+
public ResourceManagerWrapper(int i) {
super(ResourceManagerWrapper.class.getName() + "_" + i);
index = i;
@@ -448,10 +451,11 @@ public class MiniYARNCluster extends CompositeService {
@Override
protected synchronized void serviceStart() throws Exception {
startResourceManager(index);
+ Configuration conf = resourceManagers[index].getConfig();
LOG.info("MiniYARN ResourceManager address: " +
- getConfig().get(YarnConfiguration.RM_ADDRESS));
- LOG.info("MiniYARN ResourceManager web address: " +
- WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
+ conf.get(YarnConfiguration.RM_ADDRESS));
+ LOG.info("MiniYARN ResourceManager web address: " + WebAppUtils
+ .getRMWebAppURLWithoutScheme(conf));
super.serviceStart();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d39b344/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
index 8a3c9e7..3e35bd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
@@ -19,7 +19,9 @@
package org.apache.hadoop.yarn.server;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.HATestUtil;
import org.junit.Assert;
import org.junit.Test;
@@ -112,4 +114,39 @@ public class TestMiniYarnCluster {
}
}
}
+
+ @Test
+ public void testMultiRMConf() {
+ String RM1_NODE_ID = "rm1", RM2_NODE_ID = "rm2";
+ int RM1_PORT_BASE = 10000, RM2_PORT_BASE = 20000;
+ Configuration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
+ conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
+ HATestUtil.setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE, conf);
+ HATestUtil.setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE, conf);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
+
+ MiniYARNCluster cluster =
+ new MiniYARNCluster(TestMiniYarnCluster.class.getName(),
+ 2, 0, 1, 1);
+ cluster.init(conf);
+ Configuration conf1 = cluster.getResourceManager(0).getConfig(),
+ conf2 = cluster.getResourceManager(1).getConfig();
+ Assert.assertFalse(conf1 == conf2);
+ Assert.assertEquals("0.0.0.0:18032",
+ conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID)));
+ Assert.assertEquals("0.0.0.0:28032",
+ conf1.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID)));
+ Assert.assertEquals("rm1", conf1.get(YarnConfiguration.RM_HA_ID));
+
+ Assert.assertEquals("0.0.0.0:18032",
+ conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID)));
+ Assert.assertEquals("0.0.0.0:28032",
+ conf2.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID)));
+ Assert.assertEquals("rm2", conf2.get(YarnConfiguration.RM_HA_ID));
+ }
}