You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/10/07 22:39:34 UTC
svn commit: r1180214 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoo...
Author: acmurthy
Date: Fri Oct 7 20:39:33 2011
New Revision: 1180214
URL: http://svn.apache.org/viewvc?rev=1180214&view=rev
Log:
MAPREDUCE-3033. Ensure Master interface pays attention to classic v/s yarn frameworks. Contributed by Hitesh Shah.
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1180214&r1=1180213&r2=1180214&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Oct 7 20:39:33 2011
@@ -1546,6 +1546,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2751. Modified NodeManager to stop leaving around local files
after application finishes. (Siddharth Seth via vinodkv)
+ MAPREDUCE-3033. Ensure Master interface pays attention to classic v/s yarn
+ frameworks. (Hitesh Shah via acmurthy)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java?rev=1180214&r1=1180213&r2=1180214&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java Fri Oct 7 20:39:33 2011
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
public class Master {
@@ -33,20 +34,33 @@ public class Master {
}
public static String getMasterUserName(Configuration conf) {
- return conf.get(MRConfig.MASTER_USER_NAME);
+ String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+ if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
+ return conf.get(MRConfig.MASTER_USER_NAME);
+ }
+ else {
+ return conf.get(YarnConfiguration.RM_PRINCIPAL);
+ }
}
public static InetSocketAddress getMasterAddress(Configuration conf) {
- String jobTrackerStr =
- conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
- return NetUtils.createSocketAddr(jobTrackerStr);
+ String masterAddress;
+ String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+ if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
+ masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
+ }
+ else {
+ masterAddress = conf.get(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS);
+ }
+ return NetUtils.createSocketAddr(masterAddress);
}
public static String getMasterPrincipal(Configuration conf)
throws IOException {
- String jtHostname = getMasterAddress(conf).getHostName();
- // get jobtracker principal for use as delegation token renewer
- return SecurityUtil.getServerPrincipal(getMasterUserName(conf), jtHostname);
+ String masterHostname = getMasterAddress(conf).getHostName();
+ // get kerberos principal for use as delegation token renewer
+ return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname);
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java?rev=1180214&r1=1180213&r2=1180214&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java Fri Oct 7 20:39:33 2011
@@ -343,7 +343,7 @@ public class ReduceTask extends Task {
boolean isLocal = false;
// local iff framework == classic && master address == local
- String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
+ String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
isLocal = "local".equals(job.get(MRConfig.MASTER_ADDRESS, "local"));
}
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java?rev=1180214&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java Fri Oct 7 20:39:33 2011
@@ -0,0 +1,88 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapred;
+
+import static org.junit.Assert.*;
+
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Test;
+
+public class TestMaster {
+
+ @Test
+ public void testGetMasterAddress() {
+ YarnConfiguration conf = new YarnConfiguration();
+
+ // Default is yarn framework
+ String masterHostname = Master.getMasterAddress(conf).getHostName();
+
+ // no address set so should default to default rm address
+ InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS);
+ assertEquals(masterHostname, rmAddr.getHostName());
+
+ // Trying invalid master address for classic
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
+ conf.set(MRConfig.MASTER_ADDRESS, "local");
+
+ // should throw an exception for invalid value
+ try {
+ Master.getMasterAddress(conf);
+ fail("Should not reach here as there is a bad master address");
+ }
+ catch (Exception e) {
+ // Expected
+ }
+
+ // Change master address to a valid value
+ conf.set(MRConfig.MASTER_ADDRESS, "bar.com:9999");
+ masterHostname = Master.getMasterAddress(conf).getHostName();
+ assertEquals(masterHostname, "bar.com");
+
+ // change framework to yarn
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+ conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192");
+ masterHostname = Master.getMasterAddress(conf).getHostName();
+ assertEquals(masterHostname, "foo1.com");
+
+ }
+
+ @Test
+ public void testGetMasterUser() {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(MRConfig.MASTER_USER_NAME, "foo");
+ conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");
+
+ // default is yarn framework
+ assertEquals(Master.getMasterUserName(conf), "bar");
+
+ // set framework name to classic
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
+ assertEquals(Master.getMasterUserName(conf), "foo");
+
+ // change framework to yarn
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+ assertEquals(Master.getMasterUserName(conf), "bar");
+
+ }
+
+}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1180214&r1=1180213&r2=1180214&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java Fri Oct 7 20:39:33 2011
@@ -186,7 +186,7 @@ public class TestClientServiceDelegate {
private ClientServiceDelegate getClientServiceDelegate(
MRClientProtocol historyServerProxy, ResourceMgrDelegate rm) {
Configuration conf = new YarnConfiguration();
- conf.set(MRConfig.FRAMEWORK_NAME, "yarn");
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
ClientServiceDelegate clientServiceDelegate = new ClientServiceDelegate(
conf, rm, oldJobId, historyServerProxy);
return clientServiceDelegate;