You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ra...@apache.org on 2011/10/12 14:19:02 UTC
svn commit: r1182330 - in /hadoop/common/trunk/hadoop-mapreduce-project:
CHANGES.txt ivy.xml
src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
Author: ravigummadi
Date: Wed Oct 12 12:19:02 2011
New Revision: 1182330
URL: http://svn.apache.org/viewvc?rev=1182330&view=rev
Log:
MAPREDUCE-3166. [Rumen] Make Rumen use job history api instead of relying on current history file name format.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/ivy.xml
hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1182330&r1=1182329&r2=1182330&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Oct 12 12:19:02 2011
@@ -24,6 +24,9 @@ Trunk (unreleased changes)
BUG FIXES
+ MAPREDUCE-3166. [Rumen] Make Rumen use job history api instead of relying
+ on current history file name format. (Ravi Gummadi)
+
MAPREDUCE-3157. [Rumen] Fix TraceBuilder to handle 0.20 history file
names also. (Ravi Gummadi)
Modified: hadoop/common/trunk/hadoop-mapreduce-project/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/ivy.xml?rev=1182330&r1=1182329&r2=1182330&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/ivy.xml (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/ivy.xml Wed Oct 12 12:19:02 2011
@@ -87,6 +87,8 @@
rev="${yarn.version}" conf="compile->default"/>
<dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core"
rev="${yarn.version}" conf="compile->default"/>
+ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common"
+ rev="${yarn.version}" conf="compile->default"/>
<dependency org="org.apache.hadoop" name="hadoop-yarn-common"
rev="${yarn.version}" conf="compile->default"/>
<dependency org="log4j" name="log4j" rev="${log4j.version}"
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java?rev=1182330&r1=1182329&r2=1182330&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/tools/rumen/TestRumenJobTraces.java Wed Oct 12 12:19:02 2011
@@ -310,7 +310,7 @@ public class TestRumenJobTraces {
public void testJobHistoryFilenameParsing() throws IOException {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
- String user = "testUser";
+
org.apache.hadoop.mapred.JobID jid =
new org.apache.hadoop.mapred.JobID("12345", 1);
final Path rootInputDir =
@@ -318,7 +318,8 @@ public class TestRumenJobTraces {
.makeQualified(lfs.getUri(), lfs.getWorkingDirectory());
// Check if current jobhistory filenames are detected properly
- Path jhFilename = JobHistory.getJobHistoryFile(rootInputDir, jid, user);
+ Path jhFilename = org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils
+ .getStagingJobHistoryFile(rootInputDir, jid.toString(), 1);
validateHistoryFileNameParsing(jhFilename, jid);
// Check if Pre21 V1 jophistory file names are detected properly
Modified: hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java?rev=1182330&r1=1182329&r2=1182330&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/src/tools/org/apache/hadoop/tools/rumen/JobHistoryUtils.java Wed Oct 12 12:19:02 2011
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.tools.rumen;
+import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -57,6 +58,30 @@ public class JobHistoryUtils {
}
/**
+ * Extracts job id from the current hadoop version's job history file name.
+ * @param fileName job history file name from which job id is to be extracted
+ * @return job id if the history file name format is same as that of the
+ * current hadoop version. Returns null otherwise.
+ */
+ private static String extractJobIDFromCurrentHistoryFile(String fileName) {
+ JobID id = null;
+ if (org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils
+ .isValidJobHistoryFileName(fileName)) {
+ try {
+ id = org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils
+ .getJobIDFromHistoryFilePath(fileName);
+ } catch (IOException e) {
+ // Ignore this exception and go ahead with getting of jobID assuming
+ // older hadoop verison's history file
+ }
+ }
+ if (id != null) {
+ return id.toString();
+ }
+ return null;
+ }
+
+ /**
* Extracts jobID string from the given job history file name.
* @param fileName name of the job history file
* @return JobID if the given <code>fileName</code> is a valid job history
@@ -67,16 +92,22 @@ public class JobHistoryUtils {
// (1) old pre21 job history file name format
// (2) new pre21 job history file name format
// (3) current job history file name format i.e. 0.22
+
+ // Try to get the jobID assuming that the history file is from the current
+ // hadoop version
+ String jobID = extractJobIDFromCurrentHistoryFile(fileName);
+ if (jobID != null) {
+ return jobID;//history file is of current hadoop version
+ }
+
+ // History file could be of older hadoop versions
String pre21JobID = applyParser(fileName,
Pre21JobHistoryConstants.JOBHISTORY_FILENAME_REGEX_V1);
if (pre21JobID == null) {
pre21JobID = applyParser(fileName,
Pre21JobHistoryConstants.JOBHISTORY_FILENAME_REGEX_V2);
}
- if (pre21JobID != null) {
- return pre21JobID;
- }
- return applyParser(fileName, JobHistory.JOBHISTORY_FILENAME_REGEX);
+ return pre21JobID;
}
/**