You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2009/10/18 21:38:53 UTC
svn commit: r826509 - in /hadoop/mapreduce/branches/branch-0.21: ./
src/test/mapred/org/apache/hadoop/tools/rumen/
src/test/tools/data/rumen/small-trace-test/
src/tools/org/apache/hadoop/tools/rumen/
Author: cdouglas
Date: Sun Oct 18 19:38:52 2009
New Revision: 826509
URL: http://svn.apache.org/viewvc?rev=826509&view=rev
Log:
MAPREDUCE-1077. Fix Rumen so that truncated tasks do not mark the job as
successful. Contributed by Dick King
Added:
hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz (with props)
hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz (with props)
Removed:
hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output
hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs
Modified:
hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java
hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java
hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java
hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java
hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=826509&r1=826508&r2=826509&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Sun Oct 18 19:38:52 2009
@@ -758,3 +758,6 @@
MAPREDUCE-1061. Add unit test validating byte specifications for gridmix
jobs. (cdouglas)
+
+ MAPREDUCE-1077. Fix Rumen so that truncated tasks do not mark the job as
+ successful. (Dick King via cdouglas)
Modified: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=826509&r1=826508&r2=826509&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java Sun Oct 18 19:38:52 2009
@@ -33,8 +33,8 @@
public class TestRumenJobTraces {
@Test
public void testSmallTrace() throws Exception {
- performSingleTest("sample-job-tracker-logs",
- "job-tracker-logs-topology-output", "job-tracker-logs-trace-output");
+ performSingleTest("sample-job-tracker-logs.gz",
+ "job-tracker-logs-topology-output", "job-tracker-logs-trace-output.gz");
}
@Test
@@ -96,9 +96,8 @@
static private <T extends DeepCompare> void jsonFileMatchesGold(
FileSystem lfs, Path result, Path gold, Class<? extends T> clazz,
String fileDescription) throws IOException {
- InputStream goldStream = lfs.open(gold);
JsonObjectMapperParser<T> goldParser =
- new JsonObjectMapperParser<T>(goldStream, clazz);
+ new JsonObjectMapperParser<T>(gold, clazz, new Configuration());
InputStream resultStream = lfs.open(result);
JsonObjectMapperParser<T> resultParser =
new JsonObjectMapperParser<T>(resultStream, clazz);
Added: hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz?rev=826509&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/job-tracker-logs-trace-output.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz?rev=826509&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/tools/data/rumen/small-trace-test/sample-job-tracker-logs.gz
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java?rev=826509&r1=826508&r2=826509&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedJob.java Sun Oct 18 19:38:52 2009
@@ -22,6 +22,7 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import java.util.TreeSet;
import org.codehaus.jackson.annotate.JsonAnySetter;
@@ -44,7 +45,8 @@
VERY_LOW, LOW, NORMAL, HIGH, VERY_HIGH
};
- static private TreeSet<String> alreadySeenAnySetterAttributes = new TreeSet<String>();
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
String jobID;
String user;
@@ -59,7 +61,7 @@
int heapMegabytes = -1;
int totalMaps = -1;
int totalReduces = -1;
- Pre21JobHistoryConstants.Values outcome = Pre21JobHistoryConstants.Values.SUCCESS;
+ Pre21JobHistoryConstants.Values outcome = null;
JobType jobtype = JobType.JAVA;
JobPriority priority = JobPriority.NORMAL;
@@ -100,7 +102,8 @@
setJobID(jobID);
}
- @SuppressWarnings("unused") // for input parameter ignored.
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
@JsonAnySetter
public void setUnknownAttribute(String attributeName, Object ignored) {
if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
@@ -388,8 +391,9 @@
}
}
- private void compare1(Pre21JobHistoryConstants.Values c1, Pre21JobHistoryConstants.Values c2,
- TreePath loc, String eltname) throws DeepInequalityException {
+ private void compare1(Pre21JobHistoryConstants.Values c1,
+ Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
if (c1 != c2) {
throw new DeepInequalityException(eltname + " miscompared", new TreePath(
loc, eltname));
Modified: hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java?rev=826509&r1=826508&r2=826509&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedLocation.java Sun Oct 18 19:38:52 2009
@@ -19,6 +19,10 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
/**
* A {@link LoggedLocation} is a representation of a point in an hierarchical
@@ -41,9 +45,12 @@
* The full path from the root of the network to the host.
*
* NOTE that this assumes that the network topology is a tree.
- */
+ */
List<String> layers = new ArrayList<String>();
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
public List<String> getLayers() {
return layers;
}
@@ -52,6 +59,17 @@
this.layers = layers;
}
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
// I'll treat this as an atomic object type
private void compareStrings(List<String> c1, List<String> c2, TreePath loc,
String eltname) throws DeepInequalityException {
Modified: hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java?rev=826509&r1=826508&r2=826509&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedNetworkTopology.java Sun Oct 18 19:38:52 2009
@@ -23,9 +23,13 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.ArrayList;
import java.util.Comparator;
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
/**
* A {@link LoggedNetworkTopology} represents a tree that in turn represents a
* hierarchy of hosts. The current version requires the tree to have all leaves
@@ -39,10 +43,24 @@
String name;
List<LoggedNetworkTopology> children = new ArrayList<LoggedNetworkTopology>();
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
public LoggedNetworkTopology() {
super();
}
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
/**
* We need this because we have to sort the {@code children} field. That field
* is set-valued, but if we sort these fields we ensure that comparisons won't
@@ -70,7 +88,8 @@
this.children = null;
if (level < ParsedHost.numberOfDistances() - 1) {
- HashMap<String, HashSet<ParsedHost>> topologies = new HashMap<String, HashSet<ParsedHost>>();
+ HashMap<String, HashSet<ParsedHost>> topologies =
+ new HashMap<String, HashSet<ParsedHost>>();
Iterator<ParsedHost> iter = hosts.iterator();
Modified: hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java?rev=826509&r1=826508&r2=826509&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedSingleRelativeRanking.java Sun Oct 18 19:38:52 2009
@@ -17,6 +17,11 @@
*/
package org.apache.hadoop.tools.rumen;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
/**
* A {@link LoggedSingleRelativeRanking} represents an X-Y coordinate of a
* single point in a discrete CDF.
@@ -36,6 +41,20 @@
*/
long datum = -1L;
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
public double getRelativeRanking() {
return relativeRanking;
}
Modified: hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java?rev=826509&r1=826508&r2=826509&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTask.java Sun Oct 18 19:38:52 2009
@@ -19,6 +19,10 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
/**
* A {@link LoggedTask} represents a [hadoop] task that is part of a hadoop job.
@@ -41,11 +45,26 @@
Pre21JobHistoryConstants.Values taskStatus;
List<LoggedTaskAttempt> attempts = new ArrayList<LoggedTaskAttempt>();
- ArrayList<LoggedLocation> preferredLocations = new ArrayList<LoggedLocation>();
+ ArrayList<LoggedLocation> preferredLocations =
+ new ArrayList<LoggedLocation>();
int numberMaps = -1;
int numberReduces = -1;
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
LoggedTask() {
super();
}
@@ -173,8 +192,9 @@
}
}
- private void compare1(Pre21JobHistoryConstants.Values c1, Pre21JobHistoryConstants.Values c2,
- TreePath loc, String eltname) throws DeepInequalityException {
+ private void compare1(Pre21JobHistoryConstants.Values c1,
+ Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
if (c1 == null && c2 == null) {
return;
}
Modified: hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java?rev=826509&r1=826508&r2=826509&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/tools/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java Sun Oct 18 19:38:52 2009
@@ -18,6 +18,11 @@
package org.apache.hadoop.tools.rumen;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.codehaus.jackson.annotate.JsonAnySetter;
+
// HACK ALERT!!! This "should" have have two subclasses, which might be called
// LoggedMapTaskAttempt and LoggedReduceTaskAttempt, but
// the Jackson implementation of JSON doesn't handle a
@@ -63,6 +68,20 @@
super();
}
+ static private Set<String> alreadySeenAnySetterAttributes =
+ new TreeSet<String>();
+
+ @SuppressWarnings("unused")
+ // for input parameter ignored.
+ @JsonAnySetter
+ public void setUnknownAttribute(String attributeName, Object ignored) {
+ if (!alreadySeenAnySetterAttributes.contains(attributeName)) {
+ alreadySeenAnySetterAttributes.add(attributeName);
+ System.err.println("In LoggedJob, we saw the unknown attribute "
+ + attributeName + ".");
+ }
+ }
+
public long getShuffleFinished() {
return shuffleFinished;
}
@@ -259,8 +278,9 @@
}
}
- private void compare1(Pre21JobHistoryConstants.Values c1, Pre21JobHistoryConstants.Values c2,
- TreePath loc, String eltname) throws DeepInequalityException {
+ private void compare1(Pre21JobHistoryConstants.Values c1,
+ Pre21JobHistoryConstants.Values c2, TreePath loc, String eltname)
+ throws DeepInequalityException {
if (c1 != c2) {
throw new DeepInequalityException(eltname + " miscompared", new TreePath(
loc, eltname));