You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/10/07 22:45:18 UTC
svn commit: r1180222 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/
hadoop-mapreduce-client/hadoop-mapreduce-client-c...
Author: acmurthy
Date: Fri Oct 7 20:45:18 2011
New Revision: 1180222
URL: http://svn.apache.org/viewvc?rev=1180222&view=rev
Log:
Merge -c 1180220 from trunk to branch-0.23 to fix MAPREDUCE-2802.
Added:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/
- copied from r1180220, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
- copied unchanged from r1180220, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1180222&r1=1180221&r2=1180222&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Oct 7 20:45:18 2011
@@ -1512,6 +1512,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3033. Ensure Master interface pays attention to classic v/s yarn
frameworks. (Hitesh Shah via acmurthy)
+ MAPREDUCE-2802. Ensure JobHistory filenames have jobId. (Jonathan Eagles
+ via acmurthy)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java?rev=1180222&r1=1180221&r2=1180222&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java Fri Oct 7 20:45:18 2011
@@ -29,11 +29,11 @@ import org.apache.hadoop.mapreduce.v2.ap
public class FileNameIndexUtils {
- static final String UNDERSCORE_ESCAPE = "%5F";
static final int JOB_NAME_TRIM_LENGTH = 50;
- //This has to be underscore currently. Untill escape uses DELIMITER.
- static final String DELIMITER = "_";
+ // Sanitize job history file for predictable parsing
+ static final String DELIMITER = "-";
+ static final String DELIMITER_ESCAPE = "%2D";
private static final int JOB_ID_INDEX = 0;
private static final int SUBMIT_TIME_INDEX = 1;
@@ -54,7 +54,7 @@ public class FileNameIndexUtils {
public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException {
StringBuilder sb = new StringBuilder();
//JobId
- sb.append(escapeUnderscores(TypeConverter.fromYarn(indexInfo.getJobId()).toString()));
+ sb.append(escapeDelimiters(TypeConverter.fromYarn(indexInfo.getJobId()).toString()));
sb.append(DELIMITER);
//StartTime
@@ -62,11 +62,11 @@ public class FileNameIndexUtils {
sb.append(DELIMITER);
//UserName
- sb.append(escapeUnderscores(getUserName(indexInfo)));
+ sb.append(escapeDelimiters(getUserName(indexInfo)));
sb.append(DELIMITER);
//JobName
- sb.append(escapeUnderscores(trimJobName(getJobName(indexInfo))));
+ sb.append(escapeDelimiters(trimJobName(getJobName(indexInfo))));
sb.append(DELIMITER);
//FinishTime
@@ -136,13 +136,13 @@ public class FileNameIndexUtils {
*/
public static String encodeJobHistoryFileName(String logFileName)
throws IOException {
- String replacementUnderscoreEscape = null;
+ String replacementDelimiterEscape = null;
- if (logFileName.contains(UNDERSCORE_ESCAPE)) {
- replacementUnderscoreEscape = nonOccursString(logFileName);
+ // Temporarily protect the escape delimiters from encoding
+ if (logFileName.contains(DELIMITER_ESCAPE)) {
+ replacementDelimiterEscape = nonOccursString(logFileName);
- logFileName = replaceStringInstances
- (logFileName, UNDERSCORE_ESCAPE, replacementUnderscoreEscape);
+ logFileName = logFileName.replaceAll(DELIMITER_ESCAPE, replacementDelimiterEscape);
}
String encodedFileName = null;
@@ -154,10 +154,10 @@ public class FileNameIndexUtils {
ioe.setStackTrace(uee.getStackTrace());
throw ioe;
}
-
- if (replacementUnderscoreEscape != null) {
- encodedFileName = replaceStringInstances
- (encodedFileName, replacementUnderscoreEscape, UNDERSCORE_ESCAPE);
+
+ // Restore protected escape delimiters after encoding
+ if (replacementDelimiterEscape != null) {
+ encodedFileName = encodedFileName.replaceAll(replacementDelimiterEscape, DELIMITER_ESCAPE);
}
return encodedFileName;
@@ -214,29 +214,10 @@ public class FileNameIndexUtils {
return in;
}
- private static String escapeUnderscores(String escapee) {
- return replaceStringInstances(escapee, "_", UNDERSCORE_ESCAPE);
+ private static String escapeDelimiters(String escapee) {
+ return escapee.replaceAll(DELIMITER, DELIMITER_ESCAPE);
}
-
- // I tolerate this code because I expect a low number of
- // occurrences in a relatively short string
- private static String replaceStringInstances
- (String logFileName, String old, String replacement) {
- int index = logFileName.indexOf(old);
-
- while (index > 0) {
- logFileName = (logFileName.substring(0, index)
- + replacement
- + replaceStringInstances
- (logFileName.substring(index + old.length()),
- old, replacement));
- index = logFileName.indexOf(old);
- }
-
- return logFileName;
- }
-
/**
* Trims the job-name if required
*/