You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 07:00:23 UTC
svn commit: r1079250 - in /hadoop/mapreduce/branches/yahoo-merge/src:
java/org/apache/hadoop/mapred/
java/org/apache/hadoop/mapreduce/server/tasktracker/
test/mapred/org/apache/hadoop/mapred/
test/mapred/org/apache/hadoop/mapreduce/lib/chain/
Author: omalley
Date: Tue Mar 8 06:00:22 2011
New Revision: 1079250
URL: http://svn.apache.org/viewvc?rev=1079250&view=rev
Log:
commit 4813f70f71ac02ad153d977716519c6cab0f3663
Author: Greg Roelofs <ro...@yahoo-inc.com>
Date: Wed Jan 19 17:12:36 2011 -0800
Next round of uber-fixes (5 unit tests and typo-fixes only), corresponding to
changes in v10b patch that were not already in v10.
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java Tue Mar 8 06:00:22 2011
@@ -54,7 +54,7 @@ class TaskMemoryManagerThread extends Th
private final List<TaskAttemptID> tasksToBeRemoved;
private static final String MEMORY_USAGE_STRING =
- "Memory usage of ProcessTree %s for task-id %s : Virutal %d bytes, " +
+ "Memory usage of ProcessTree %s for task-id %s : Virtual %d bytes, " +
"limit : %d bytes; Physical %d bytes, limit %d bytes";
public TaskMemoryManagerThread(TaskTracker taskTracker) {
@@ -299,7 +299,7 @@ class TaskMemoryManagerThread extends Th
if (doCheckVirtualMemory() &&
memoryStillInUsage > maxMemoryAllowedForAllTasks) {
LOG.warn("The total memory in usage " + memoryStillInUsage
- + " is still overflowing TTs limits "
+ + " is still overflowing TT's limits "
+ maxMemoryAllowedForAllTasks
+ ". Trying to kill a few tasks with the least progress.");
killTasksWithLeastProgress(memoryStillInUsage);
@@ -308,7 +308,7 @@ class TaskMemoryManagerThread extends Th
if (doCheckPhysicalMemory() &&
rssMemoryStillInUsage > maxRssMemoryAllowedForAllTasks) {
LOG.warn("The total physical memory in usage " + rssMemoryStillInUsage
- + " is still overflowing TTs limits "
+ + " is still overflowing TT's limits "
+ maxRssMemoryAllowedForAllTasks
+ ". Trying to kill a few tasks with the highest memory.");
killTasksWithMaxRssMemory(rssMemoryStillInUsage);
@@ -440,8 +440,8 @@ class TaskMemoryManagerThread extends Th
killTask(tid, msg);
}
} else {
- LOG.info("The total memory usage is overflowing TTs limits. "
- + "But found no alive task to kill for freeing memory.");
+ LOG.info("The total memory usage is overflowing TT's limits, "
+ + "but found no live task to kill to free memory.");
}
}
@@ -457,8 +457,8 @@ class TaskMemoryManagerThread extends Th
}
/**
- * Starting from the tasks use the highest amount of RSS memory,
- * kill the tasks until the RSS memory meets the requirement
+ * Starting from the tasks that use the highest amount of RSS memory,
+ * kill the tasks until the RSS memory meets the requirement.
* @param rssMemoryInUsage
*/
private void killTasksWithMaxRssMemory(long rssMemoryInUsage) {
@@ -500,8 +500,8 @@ class TaskMemoryManagerThread extends Th
killTask(tid, msg);
}
} else {
- LOG.info("The total physical memory usage is overflowing TTs limits. "
- + "But found no alive task to kill for freeing memory.");
+ LOG.info("The total physical memory usage is overflowing TT's limits, "
+ + "but found no live task to kill to free memory.");
}
}
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java Tue Mar 8 06:00:22 2011
@@ -90,9 +90,10 @@ public interface TTConfig extends MRConf
"mapreduce.tasktracker.cache.local.numberdirectories";
public static final String TT_OUTOFBAND_HEARBEAT =
"mapreduce.tasktracker.outofband.heartbeat";
- public static final String TT_RESERVED_PHYSCIALMEMORY_MB =
+ public static final String TT_RESERVED_PHYSICALMEMORY_MB =
"mapreduce.tasktracker.reserved.physicalmemory.mb";
- public static final String TT_USER_NAME = "mapreduce.tasktracker.kerberos.principal";
+ public static final String TT_USER_NAME =
+ "mapreduce.tasktracker.kerberos.principal";
public static final String TT_KEYTAB_FILE =
"mapreduce.tasktracker.keytab.file";
public static final String TT_GROUP =
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java Tue Mar 8 06:00:22 2011
@@ -41,6 +41,11 @@ public class TestJvmReuse {
* A mapper class in which all attempts log taskid. Zeroth attempt of task
* with id=taskWithCleanup, fails with System.exit to force a cleanup attempt
* for the task in a new jvm.
+ *
+ * Note that this approach is fundamentally incompatible with uberization
+ * since sub-MapTasks have "fake"/virtual task attempt IDs that are _always_
+ * "zeroth" attempt. (And sub-MapTasks have no visibility of UberTask's
+ * attempt ID.)
*/
public static class MapperClass extends MapReduceBase implements
Mapper<LongWritable, Text, Text, IntWritable> {
@@ -93,6 +98,8 @@ public class TestJvmReuse {
FileOutputFormat.setOutputPath(conf, outDir);
// enable jvm reuse
conf.setNumTasksToExecutePerJvm(-1);
+ // disable uberization (see uber comment in MapperClass above)
+ conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
// return the RunningJob handle.
return new JobClient(conf).submitJob(conf);
}
@@ -118,7 +125,8 @@ public class TestJvmReuse {
// validates logs of all attempts of the job.
private void validateJob(RunningJob job, MiniMRCluster mr) throws IOException {
- assertEquals(JobStatus.SUCCEEDED, job.getJobState());
+ assertEquals("JobStatus not SUCCEEDED", JobStatus.SUCCEEDED,
+ job.getJobState());
long uses = job.getCounters().findCounter("jvm", "use").getValue();
assertTrue("maps = " + numMappers + ", jvms = " + uses, numMappers < uses);
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java Tue Mar 8 06:00:22 2011
@@ -117,6 +117,10 @@ public class TestTaskFail extends TestCa
FileInputFormat.setInputPaths(conf, inDir);
FileOutputFormat.setOutputPath(conf, outDir);
conf.setSpeculativeExecution(false);
+ // mapper class explicitly depends on being able to see "real" taskids, but
+ // uberized map (sub-)taskids never change => subtask (and ubertask, and
+ // job) always fails due to being attempt "_0"
+ conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
String TEST_ROOT_DIR = new Path(System.getProperty("test.build.data",
"/tmp")).toString().replace(' ', '+');
conf.set("test.build.data", TEST_ROOT_DIR);
@@ -164,8 +168,8 @@ public class TestTaskFail extends TestCa
private void validateJob(RunningJob job, JobTracker jt)
throws IOException {
- assertEquals(JobStatus.SUCCEEDED, job.getJobState());
-
+ assertEquals("job not successful", JobStatus.SUCCEEDED, job.getJobState());
+
JobID jobId = job.getID();
// construct the task id of first map task
// this should not be cleanup attempt since the first attempt
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java Tue Mar 8 06:00:22 2011
@@ -96,6 +96,9 @@ public class TestTaskPerformanceSplitTra
try {
JobConf jConf = mrCluster.createJobConf();
+ // disable uberization (MR-1220) until JobHistory supports uber-specific
+ // events with complete split info (for Rumen network topology)
+ jConf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
// construct a job with 1 map and 1 reduce task.
Job job = MapReduceTestUtil.createJob(jConf, inDir, outDir, 1, 1);
// disable setup/cleanup
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java Tue Mar 8 06:00:22 2011
@@ -183,7 +183,7 @@ public class TestTaskTrackerMemoryManage
fConf.setLong(MRConfig.MAPMEMORY_MB, 2 * 1024L);
fConf.setLong(MRConfig.REDUCEMEMORY_MB, 2 * 1024L);
// Reserve only 1 mb of the memory on TaskTrackers
- fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB, 1L);
+ fConf.setLong(TTConfig.TT_RESERVED_PHYSICALMEMORY_MB, 1L);
startCluster(new JobConf());
JobConf conf = new JobConf(miniMRCluster.createJobConf());
@@ -240,7 +240,7 @@ public class TestTaskTrackerMemoryManage
// very small value, so that no task escapes to successful completion.
fConf.setInt(TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL, 100);
// Reserve only 1 mb of the memory on TaskTrackers
- fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB, 1L);
+ fConf.setLong(TTConfig.TT_RESERVED_PHYSICALMEMORY_MB, 1L);
startCluster(fConf);
runJobExceedingMemoryLimit(true);
}
@@ -547,10 +547,11 @@ public class TestTaskTrackerMemoryManage
new LinuxResourceCalculatorPlugin();
long totalPhysicalMemory = memoryCalculatorPlugin.getPhysicalMemorySize();
long reservedPhysicalMemory = totalPhysicalMemory / (1024 * 1024) + 1;
- fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB,
+ fConf.setLong(TTConfig.TT_RESERVED_PHYSICALMEMORY_MB,
reservedPhysicalMemory);
long maxRssMemoryAllowedForAllTasks = totalPhysicalMemory -
reservedPhysicalMemory * 1024 * 1024L;
+ // must match string near line 500 of TaskMemoryManagerThread.java:
Pattern physicalMemoryOverLimitPattern = Pattern.compile(
"Killing one of the memory-consuming tasks - .*"
+ ", as the cumulative RSS memory usage of all the tasks on "
@@ -571,13 +572,20 @@ public class TestTaskTrackerMemoryManage
// Start the job
Job job = sleepJob.createJob(1, 1, 100000, 1, 100000, 1);
job.submit();
+ JobInProgress jip = miniMRCluster.getJobTrackerRunner().getJobTracker()
+ .getJob(JobID.downgrade(job.getJobID()));
boolean TTOverFlowMsgPresent = false;
while (true) {
List<TaskReport> allTaskReports = new ArrayList<TaskReport>();
- allTaskReports.addAll(Arrays.asList(jClient
- .getSetupTaskReports(JobID.downgrade(job.getJobID()))));
- allTaskReports.addAll(Arrays.asList(jClient
- .getMapTaskReports(JobID.downgrade(job.getJobID()))));
+ if (jip.isUber()) {
+ allTaskReports.addAll(Arrays.asList(
+ jClient.getReduceTaskReports(JobID.downgrade(job.getJobID()))));
+ } else {
+ allTaskReports.addAll(Arrays.asList(
+ jClient.getSetupTaskReports(JobID.downgrade(job.getJobID()))));
+ allTaskReports.addAll(Arrays.asList(
+ jClient.getMapTaskReports(JobID.downgrade(job.getJobID()))));
+ }
for (TaskReport tr : allTaskReports) {
String[] diag = tr.getDiagnostics();
for (String str : diag) {
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java Tue Mar 8 06:00:22 2011
@@ -201,23 +201,27 @@ public class TestWebUIAuthorization exte
return job;
}
- // Waits till the map task gets started and gets its tipId from map reports
- // and returns the tipId
+ // Waits until the map or uber task gets started; gets its tipId from task
+ // reports; and returns the tipId.
private TaskID getTIPId(MiniMRCluster cluster,
org.apache.hadoop.mapreduce.JobID jobid) throws Exception {
JobClient client = new JobClient(cluster.createJobConf());
JobID jobId = JobID.downgrade(jobid);
- TaskReport[] mapReports = null;
+ JobInProgress jip =
+ cluster.getJobTrackerRunner().getJobTracker().getJob(jobId);
+ TaskReport[] taskReports = null;
TaskID tipId = null;
do { // make sure that the map task is running
Thread.sleep(200);
- mapReports = client.getMapTaskReports(jobId);
- } while (mapReports.length == 0);
+ taskReports = jip.isUber()
+ ? client.getReduceTaskReports(jobId)
+ : client.getMapTaskReports(jobId);
+ } while (taskReports.length == 0);
- for (TaskReport r : mapReports) {
+ for (TaskReport r : taskReports) {
tipId = r.getTaskID();
- break;// because we have only one map
+ break;// because we have only one map (or one reduce if uberized)
}
return tipId;
}
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java Tue Mar 8 06:00:22 2011
@@ -139,7 +139,7 @@ public class TestChainErrors extends Had
*
* @throws Exception
*/
- public void testChainMapNoOuptut() throws Exception {
+ public void testChainMapNoOutput() throws Exception {
Configuration conf = createJobConf();
String expectedOutput = "";
@@ -163,7 +163,7 @@ public class TestChainErrors extends Had
*
* @throws Exception
*/
- public void testChainReduceNoOuptut() throws Exception {
+ public void testChainReduceNoOutput() throws Exception {
Configuration conf = createJobConf();
String expectedOutput = "";