You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/09/26 09:08:49 UTC
[31/50] [abbrv] hadoop git commit: MAPREDUCE-6632.
Master.getMasterAddress() should be updated to use YARN-4629 (templedf via
rkanter)
MAPREDUCE-6632. Master.getMasterAddress() should be updated to use YARN-4629 (templedf via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4fc632ae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4fc632ae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4fc632ae
Branch: refs/heads/MAPREDUCE-6608
Commit: 4fc632ae19a1d6b0ec09cc7ead789a3cab1c2f1c
Parents: 40acace
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Sep 22 16:12:56 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Sep 22 16:12:56 2016 -0700
----------------------------------------------------------------------
.../hadoop-mapreduce-client-core/pom.xml | 6 +-
.../java/org/apache/hadoop/mapred/Master.java | 70 ++++++--------------
.../org/apache/hadoop/mapred/TestMaster.java | 56 +---------------
3 files changed, 28 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 4de24c7..00bb11b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -37,7 +37,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
index d84e395..ec00405 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
@@ -19,75 +19,45 @@
package org.apache.hadoop.mapred;
import java.io.IOException;
-import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.client.util.YarnClientUtils;
@Private
@Unstable
public class Master {
-
- private static final Log LOG = LogFactory.getLog(Master.class);
-
public enum State {
INITIALIZING, RUNNING;
}
- public static String getMasterUserName(Configuration conf) {
- String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
- return conf.get(MRConfig.MASTER_USER_NAME);
- }
- else {
- return conf.get(YarnConfiguration.RM_PRINCIPAL);
- }
+ public static String getMasterAddress(Configuration conf) {
+ String masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
+
+ return NetUtils.createSocketAddr(masterAddress, 8012,
+ MRConfig.MASTER_ADDRESS).getHostName();
}
-
- public static InetSocketAddress getMasterAddress(Configuration conf) {
- String masterAddress;
- String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+
+ public static String getMasterPrincipal(Configuration conf)
+ throws IOException {
+ String masterPrincipal;
+ String framework = conf.get(MRConfig.FRAMEWORK_NAME,
+ MRConfig.YARN_FRAMEWORK_NAME);
+
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
- masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
- return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
- } else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
- HAUtil.isHAEnabled(conf)) {
- YarnConfiguration yarnConf = new YarnConfiguration(conf);
- if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
- String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
- if (rmIds != null && rmIds.length > 0) {
- // If RM_HA_ID is not configured, use the first one.
- // Because any valid RM HA ID should work.
- yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
- } else {
- LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
- }
- }
- return yarnConf.getSocketAddr(
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
+ String masterAddress = getMasterAddress(conf);
+ // get kerberos principal for use as delegation token renewer
+ masterPrincipal =
+ SecurityUtil.getServerPrincipal(conf.get(MRConfig.MASTER_USER_NAME),
+ masterAddress);
} else {
- return conf.getSocketAddr(
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
+ masterPrincipal = YarnClientUtils.getRmPrincipal(conf);
}
- }
- public static String getMasterPrincipal(Configuration conf)
- throws IOException {
- String masterHostname = getMasterAddress(conf).getHostName();
- // get kerberos principal for use as delegation token renewer
- return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname);
+ return masterPrincipal;
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
index 498abbc..0f4ebee 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
@@ -18,13 +18,10 @@
package org.apache.hadoop.mapred;
-import static org.junit.Assert.*;
-
-import java.net.InetSocketAddress;
-
import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import org.junit.Test;
public class TestMaster {
@@ -33,13 +30,6 @@ public class TestMaster {
public void testGetMasterAddress() {
YarnConfiguration conf = new YarnConfiguration();
- // Default is yarn framework
- String masterHostname = Master.getMasterAddress(conf).getHostName();
-
- // no address set so should default to default rm address
- InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS);
- assertEquals(masterHostname, rmAddr.getHostName());
-
// Trying invalid master address for classic
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
conf.set(MRConfig.MASTER_ADDRESS, "local:invalid");
@@ -55,47 +45,7 @@ public class TestMaster {
// Change master address to a valid value
conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042");
- masterHostname = Master.getMasterAddress(conf).getHostName();
+ String masterHostname = Master.getMasterAddress(conf);
assertEquals(masterHostname, "bar.com");
-
- // change framework to yarn
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192");
- masterHostname = Master.getMasterAddress(conf).getHostName();
- assertEquals(masterHostname, "foo1.com");
-
- // change framework to yarn and enable HA
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
- conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
- conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
- conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
- masterHostname = Master.getMasterAddress(conf).getHostName();
- // If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
- assertEquals(masterHostname, "rm1.com");
- conf.set(YarnConfiguration.RM_HA_ID, "rm2");
- masterHostname = Master.getMasterAddress(conf).getHostName();
- // If RM_HA_ID is configured, use the given RM_HA_ID.
- assertEquals(masterHostname, "rm2.com");
}
-
- @Test
- public void testGetMasterUser() {
- YarnConfiguration conf = new YarnConfiguration();
- conf.set(MRConfig.MASTER_USER_NAME, "foo");
- conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");
-
- // default is yarn framework
- assertEquals(Master.getMasterUserName(conf), "bar");
-
- // set framework name to classic
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
- assertEquals(Master.getMasterUserName(conf), "foo");
-
- // change framework to yarn
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- assertEquals(Master.getMasterUserName(conf), "bar");
-
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org