You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zx...@apache.org on 2015/09/24 17:12:35 UTC
hadoop git commit: MAPREDUCE-6484. Yarn Client uses local address
instead of RM address as token renewer in a secure cluster when RM HA is
enabled. Contributed by Zhihai Xu
Repository: hadoop
Updated Branches:
refs/heads/trunk e52bc697f -> 97a08807e
MAPREDUCE-6484. Yarn Client uses local address instead of RM address as token renewer in a secure cluster when RM HA is enabled. Contributed by Zhihai Xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97a08807
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97a08807
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97a08807
Branch: refs/heads/trunk
Commit: 97a08807ec328a6f455d941fe3d9a469d885caa6
Parents: e52bc69
Author: Zhihai Xu <zx...@apache.org>
Authored: Thu Sep 24 08:12:07 2015 -0700
Committer: Zhihai Xu <zx...@apache.org>
Committed: Thu Sep 24 08:12:07 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapred/Master.java | 27 +++++++++++++++++---
.../org/apache/hadoop/mapred/TestMaster.java | 13 ++++++++++
3 files changed, 40 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97a08807/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 2d750e4..4bcd5d8 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -577,6 +577,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6460. TestRMContainerAllocator.
testAttemptNotFoundCausesRMCommunicatorException fails. (Zhihai Xu)
+ MAPREDUCE-6484. Yarn Client uses local address instead of RM address as
+ token renewer in a secure cluster when RM HA is enabled. (Zhihai Xu)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97a08807/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
index 3bacc5e..d84e395 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
@@ -21,18 +21,23 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
import java.net.InetSocketAddress;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@Private
@Unstable
public class Master {
-
+
+ private static final Log LOG = LogFactory.getLog(Master.class);
+
public enum State {
INITIALIZING, RUNNING;
}
@@ -53,8 +58,24 @@ public class Master {
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
- }
- else {
+ } else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
+ HAUtil.isHAEnabled(conf)) {
+ YarnConfiguration yarnConf = new YarnConfiguration(conf);
+ if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
+ String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
+ if (rmIds != null && rmIds.length > 0) {
+ // If RM_HA_ID is not configured, use the first one.
+ // Because any valid RM HA ID should work.
+ yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
+ } else {
+ LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
+ }
+ }
+ return yarnConf.getSocketAddr(
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ } else {
return conf.getSocketAddr(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97a08807/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
index 58bbb3e..498abbc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
@@ -64,6 +64,19 @@ public class TestMaster {
masterHostname = Master.getMasterAddress(conf).getHostName();
assertEquals(masterHostname, "foo1.com");
+ // change framework to yarn and enable HA
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
+ conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
+ conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
+ masterHostname = Master.getMasterAddress(conf).getHostName();
+ // If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
+ assertEquals(masterHostname, "rm1.com");
+ conf.set(YarnConfiguration.RM_HA_ID, "rm2");
+ masterHostname = Master.getMasterAddress(conf).getHostName();
+ // If RM_HA_ID is configured, use the given RM_HA_ID.
+ assertEquals(masterHostname, "rm2.com");
}
@Test