You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2010/08/15 07:33:58 UTC
svn commit: r985617 - in /hadoop/mapreduce/branches/branch-0.21: ./
src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/
src/contrib/mumak/src/test/org/apache/hadoop/mapred/
src/contrib/streaming/src/test/org/apache/hadoop/streaming/ src/t...
Author: tomwhite
Date: Sun Aug 15 05:33:57 2010
New Revision: 985617
URL: http://svn.apache.org/viewvc?rev=985617&view=rev
Log:
Merge -r 985615:985616 from trunk to branch-0.21. Fixes: MAPREDUCE-2012.
Modified:
hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java
hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java
hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorEndToEnd.java
hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java
hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingStatus.java
hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java
Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Sun Aug 15 05:33:57 2010
@@ -1506,3 +1506,6 @@ Release 0.21.0 - 2010-08-13
MAPREDUCE-1920. Enables completed jobstatus store by default. (Tom White
via amareshwari)
+ MAPREDUCE-2012. Some contrib tests fail in branch 0.21 and trunk.
+ (Amareshwari Sriramadasu via tomwhite)
+
Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/src/test/org/apache/hadoop/mapred/TestCapacitySchedulerWithJobTracker.java Sun Aug 15 05:33:57 2010
@@ -43,6 +43,7 @@ public class TestCapacitySchedulerWithJo
clusterProps.put(TTConfig.TT_MAP_SLOTS, String.valueOf(1));
clusterProps.put(TTConfig.TT_REDUCE_SLOTS, String.valueOf(1));
clusterProps.put(JTConfig.JT_TASKS_PER_JOB, String.valueOf(1));
+ clusterProps.put(JTConfig.JT_PERSIST_JOBSTATUS, "false");
// cluster capacity 1 maps, 1 reduces
startCluster(1, clusterProps, schedulerProps);
CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
@@ -87,6 +88,7 @@ public class TestCapacitySchedulerWithJo
clusterProps.put(TTConfig.TT_MAP_SLOTS, String.valueOf(2));
clusterProps.put(TTConfig.TT_REDUCE_SLOTS, String.valueOf(2));
clusterProps.put("mapred.queue.names", queues[0] + "," + queues[1]);
+ clusterProps.put(JTConfig.JT_PERSIST_JOBSTATUS, "false");
startCluster(2, clusterProps, schedulerProps);
CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
.getTaskScheduler();
Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorDeterministicReplay.java Sun Aug 15 05:33:57 2010
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
@@ -93,6 +94,7 @@ public class TestSimulatorDeterministicR
conf.set("hadoop.log.dir", hadoopLogDir.toString());
conf.set("mapred.job.tracker.http.address",
"0.0.0.0:" + jobTrackerHttpPort);
+ conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
String[] args = { traceFile.toString(), topologyFile.toString() };
int res = ToolRunner.run(conf, new SimulatorEngine(), args);
Assert.assertEquals(0, res);
Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorEndToEnd.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorEndToEnd.java?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorEndToEnd.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorEndToEnd.java Sun Aug 15 05:33:57 2010
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.MockSimulatorEngine;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.tools.rumen.ZombieCluster;
import org.apache.hadoop.tools.rumen.ZombieJobProducer;
import org.apache.hadoop.util.ToolRunner;
@@ -56,6 +57,7 @@ public class TestSimulatorEndToEnd {
MockSimulatorEngine mockMumak = new MockSimulatorEngine(numJobs, nTrackers);
+ mumakConf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
String[] args = { traceFile.toString(), topologyFile.toString() };
int res = ToolRunner.run(conf, mockMumak, args);
Assert.assertEquals(res, 0);
Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/TestSimulatorJobTracker.java Sun Aug 15 05:33:57 2010
@@ -32,6 +32,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.mapred.TaskStatus.Phase;
import org.apache.hadoop.mapred.TaskStatus.State;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.FakeJobs;
@@ -57,6 +58,7 @@ public class TestSimulatorJobTracker {
jtConf.set("mapred.system.dir", jtConf.get("hadoop.tmp.dir", "/tmp/hadoop-"
+ jtConf.getUser())
+ "/mapred/system");
+ jtConf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
System.out.println("Created JobConf");
return jtConf;
}
Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingBadRecords.java Sun Aug 15 05:33:57 2010
@@ -71,6 +71,7 @@ public class TestStreamingBadRecords ext
protected void setUp() throws Exception {
Properties props = new Properties();
props.setProperty(JTConfig.JT_RETIREJOBS, "false");
+ props.setProperty(JTConfig.JT_PERSIST_JOBSTATUS, "false");
startCluster(true, props);
}
Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingStatus.java?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingStatus.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingStatus.java Sun Aug 15 05:33:57 2010
@@ -86,6 +86,7 @@ public class TestStreamingStatus {
FileSystem fs = null;
JobConf conf = new JobConf();
conf.setBoolean(JTConfig.JT_RETIREJOBS, false);
+ conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
try {
mr = new MiniMRCluster(1, "file:///", 3, null , null, conf);
Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/src/test/org/apache/hadoop/streaming/TestStreamingTaskLog.java Sun Aug 15 05:33:57 2010
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.TestMiniMRWithDFS;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.util.Shell;
import org.junit.Test;
@@ -71,7 +72,7 @@ public class TestStreamingTaskLog {
public void testStreamingTaskLogWithHadoopCmd() {
try {
final int numSlaves = 1;
- Configuration conf = new Configuration();
+ JobConf conf = new JobConf();
fs = FileSystem.getLocal(conf);
Path testDir = new Path(System.getProperty("test.build.data","/tmp"));
@@ -81,8 +82,9 @@ public class TestStreamingTaskLog {
fs.mkdirs(testDir);
File scriptFile = createScript(
testDir.toString() + "/testTaskLog.sh");
- mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1);
-
+ conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
+ mr = new MiniMRCluster(numSlaves, fs.getUri().toString(), 1, null, null, conf);
+
writeInputFile(fs, inputPath);
map = scriptFile.getAbsolutePath();
Modified: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java?rev=985617&r1=985616&r2=985617&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/ClusterWithLinuxTaskController.java Sun Aug 15 05:33:57 2010
@@ -175,6 +175,7 @@ public class ClusterWithLinuxTaskControl
dfsCluster = new MiniDFSCluster(conf, NUMBER_OF_NODES, true, null);
conf.set(TTConfig.TT_TASK_CONTROLLER,
MyLinuxTaskController.class.getName());
+ conf.setBoolean(JTConfig.JT_PERSIST_JOBSTATUS, false);
mrCluster =
new MiniMRCluster(NUMBER_OF_NODES, dfsCluster.getFileSystem().getUri()
.toString(), 4, null, null, conf);