You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2014/03/27 00:44:18 UTC
svn commit: r1582127 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/
hadoop-mapreduce-client/hadoop-mapreduce-client-comm...
Author: vinodkv
Date: Wed Mar 26 23:44:17 2014
New Revision: 1582127
URL: http://svn.apache.org/r1582127
Log:
MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape hyphens and thus avoid parsing errors. Contributed by Akira Ajisaka.
svn merge --ignore-ancestry -c 1582125 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1582127&r1=1582126&r2=1582127&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed Mar 26 23:44:17 2014
@@ -124,6 +124,9 @@ Release 2.4.0 - UNRELEASED
MAPREDUCE-5795. Fixed MRAppMaster to record the correct job-state after it
recovers from a commit during a previous attempt. (Xuan Gong via vinodkv)
+ MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape
+ hyphens and thus avoid parsing errors. (Akira AJISAKA via vinodkv)
+
Release 2.3.1 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java?rev=1582127&r1=1582126&r2=1582127&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java Wed Mar 26 23:44:17 2014
@@ -94,7 +94,7 @@ public class FileNameIndexUtils {
sb.append(DELIMITER);
//QueueName
- sb.append(indexInfo.getQueueName());
+ sb.append(escapeDelimiters(indexInfo.getQueueName()));
sb.append(DELIMITER);
//JobStartTime
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java?rev=1582127&r1=1582126&r2=1582127&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java Wed Mar 26 23:44:17 2014
@@ -68,6 +68,10 @@ public class TestFileNameIndexUtils {
private static final String NUM_REDUCES = "1";
private static final String JOB_STATUS = "SUCCEEDED";
private static final String QUEUE_NAME = "default";
+ private static final String QUEUE_NAME_WITH_DELIMITER = "test"
+ + FileNameIndexUtils.DELIMITER + "queue";
+ private static final String QUEUE_NAME_WITH_DELIMITER_ESCAPE = "test"
+ + FileNameIndexUtils.DELIMITER_ESCAPE + "queue";
private static final String JOB_START_TIME = "1317928742060";
@Test
@@ -192,6 +196,46 @@ public class TestFileNameIndexUtils {
}
@Test
+ public void testQueueNamePercentEncoding() throws IOException {
+ JobIndexInfo info = new JobIndexInfo();
+ JobID oldJobId = JobID.forName(JOB_ID);
+ JobId jobId = TypeConverter.toYarn(oldJobId);
+ info.setJobId(jobId);
+ info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
+ info.setUser(USER_NAME);
+ info.setJobName(JOB_NAME);
+ info.setFinishTime(Long.parseLong(FINISH_TIME));
+ info.setNumMaps(Integer.parseInt(NUM_MAPS));
+ info.setNumReduces(Integer.parseInt(NUM_REDUCES));
+ info.setJobStatus(JOB_STATUS);
+ info.setQueueName(QUEUE_NAME_WITH_DELIMITER);
+ info.setJobStartTime(Long.parseLong(JOB_START_TIME));
+
+ String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
+ Assert.assertTrue("Queue name not encoded correctly into job history file",
+ jobHistoryFile.contains(QUEUE_NAME_WITH_DELIMITER_ESCAPE));
+ }
+
+ @Test
+ public void testQueueNamePercentDecoding() throws IOException {
+ String jobHistoryFile = String.format(JOB_HISTORY_FILE_FORMATTER,
+ JOB_ID,
+ SUBMIT_TIME,
+ USER_NAME,
+ JOB_NAME,
+ FINISH_TIME,
+ NUM_MAPS,
+ NUM_REDUCES,
+ JOB_STATUS,
+ QUEUE_NAME_WITH_DELIMITER_ESCAPE,
+ JOB_START_TIME );
+
+ JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
+ Assert.assertEquals("Queue name doesn't match",
+ QUEUE_NAME_WITH_DELIMITER, info.getQueueName());
+ }
+
+ @Test
public void testJobHistoryFileNameBackwardsCompatible() throws IOException {
JobID oldJobId = JobID.forName(JOB_ID);
JobId jobId = TypeConverter.toYarn(oldJobId);