You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/10/19 19:55:08 UTC
svn commit: r1186368 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoo...
Author: vinodkv
Date: Wed Oct 19 17:55:08 2011
New Revision: 1186368
URL: http://svn.apache.org/viewvc?rev=1186368&view=rev
Log:
MAPREDUCE-3176. Fixed ant mapreduce tests that are timing out because of wrong framework name. Contributed by Hitesh Shah.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java
hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java
hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1186368&r1=1186367&r2=1186368&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Oct 19 17:55:08 2011
@@ -1683,6 +1683,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3162. Separated application-init and container-init event types
in NodeManager's Application state machine. (Todd Lipcon via vinodkv)
+ MAPREDUCE-3176. Fixed ant mapreduce tests that are timing out because
+ of wrong framework name. (Hitesh Shah via vinodkv)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java?rev=1186368&r1=1186367&r2=1186368&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ReduceTask.java Wed Oct 19 17:55:08 2011
@@ -342,9 +342,15 @@ public class ReduceTask extends Task {
RawKeyValueIterator rIter = null;
boolean isLocal = false;
- // local iff framework == local
- String framework = job.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- isLocal = framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME);
+ // local if
+ // 1) framework == local or
+ // 2) framework == null and job tracker address == local
+ String framework = job.get(MRConfig.FRAMEWORK_NAME);
+ String masterAddr = job.get(MRConfig.MASTER_ADDRESS, "local");
+ if ((framework == null && masterAddr.equals("local"))
+ || (framework != null && framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME))) {
+ isLocal = true;
+ }
if (!isLocal) {
Class combinerClass = conf.getCombinerClass();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java?rev=1186368&r1=1186367&r2=1186368&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Cluster.java Wed Oct 19 17:55:08 2011
@@ -25,6 +25,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.ServiceLoader;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -62,6 +64,7 @@ public class Cluster {
private Path sysDir = null;
private Path stagingAreaDir = null;
private Path jobHistoryDir = null;
+ private static final Log LOG = LogFactory.getLog(Cluster.class);
static {
ConfigUtil.loadResources();
@@ -83,17 +86,31 @@ public class Cluster {
for (ClientProtocolProvider provider : ServiceLoader
.load(ClientProtocolProvider.class)) {
- ClientProtocol clientProtocol = null;
- if (jobTrackAddr == null) {
- clientProtocol = provider.create(conf);
- } else {
- clientProtocol = provider.create(jobTrackAddr, conf);
- }
-
- if (clientProtocol != null) {
- clientProtocolProvider = provider;
- client = clientProtocol;
- break;
+ LOG.debug("Trying ClientProtocolProvider : "
+ + provider.getClass().getName());
+ ClientProtocol clientProtocol = null;
+ try {
+ if (jobTrackAddr == null) {
+ clientProtocol = provider.create(conf);
+ } else {
+ clientProtocol = provider.create(jobTrackAddr, conf);
+ }
+
+ if (clientProtocol != null) {
+ clientProtocolProvider = provider;
+ client = clientProtocol;
+ LOG.debug("Picked " + provider.getClass().getName()
+ + " as the ClientProtocolProvider");
+ break;
+ }
+ else {
+ LOG.info("Cannot pick " + provider.getClass().getName()
+ + " as the ClientProtocolProvider - returned null protocol");
+ }
+ }
+ catch (Exception e) {
+ LOG.info("Failed to use " + provider.getClass().getName()
+ + " due to error: " + e.getMessage());
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java?rev=1186368&r1=1186367&r2=1186368&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/TestFileSystem.java Wed Oct 19 17:55:08 2011
@@ -94,7 +94,13 @@ public class TestFileSystem extends Test
CommandFormat cf;
cf= new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
assertEquals(cf.parse(new String[] {"-get","file", "-"}, 1).get(1), "-");
- assertEquals(cf.parse(new String[] {"-get","file","-ignoreCrc","/foo"}, 1).get(1),"/foo");
+ try {
+ cf.parse(new String[] {"-get","file","-ignoreCrc","/foo"}, 1);
+ fail("Expected parsing to fail as it should stop at first non-option");
+ }
+ catch (Exception e) {
+ // Expected
+ }
cf = new CommandFormat("tail", 1, 1, "f");
assertEquals(cf.parse(new String[] {"-tail","fileName"}, 1).get(0),"fileName");
assertEquals(cf.parse(new String[] {"-tail","-f","fileName"}, 1).get(0),"fileName");
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java?rev=1186368&r1=1186367&r2=1186368&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/TestSocketFactory.java Wed Oct 19 17:55:08 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.JobClien
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.MiniMRCluster;
+import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.StandardSocketFactory;
/**
@@ -92,6 +93,7 @@ public class TestSocketFactory extends T
JobConf jconf = new JobConf(cconf);
jconf.set("mapred.job.tracker", String.format("localhost:%d",
jobTrackerPort + 10));
+ jconf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
client = new JobClient(jconf);
JobStatus[] jobs = client.jobsToComplete();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java?rev=1186368&r1=1186367&r2=1186368&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java Wed Oct 19 17:55:08 2011
@@ -96,4 +96,25 @@ public class TestClientProtocolProviderI
}
}
+ @Test
+ public void testClusterException() {
+
+ Configuration conf = new Configuration();
+ conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
+ conf.set(JTConfig.JT_IPC_ADDRESS, "local");
+
+ // initializing a cluster with this conf should throw an error.
+ // However the exception thrown should not be specific to either
+ // the job tracker client provider or the local provider
+ boolean errorThrown = false;
+ try {
+ Cluster cluster = new Cluster(conf);
+ cluster.close();
+ fail("Not expected - cluster init should have failed");
+ } catch (IOException e) {
+ errorThrown = true;
+ assert(e.getMessage().contains("Cannot initialize Cluster. Please check"));
+ }
+ assert(errorThrown);
+ }
}