You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2015/06/16 20:41:49 UTC
[30/50] [abbrv] hadoop git commit: MAPREDUCE-6350. JobHistory doesn't
support fully-functional search. Contributed by Siqi Li.
MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
Contributed by Siqi Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6785661e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6785661e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6785661e
Branch: refs/heads/HDFS-7240
Commit: 6785661e554114a4613b5fe7dabec9bfa80c41d9
Parents: 48ca23d
Author: Devaraj K <de...@apache.org>
Authored: Wed Jun 10 12:40:29 2015 +0530
Committer: Devaraj K <de...@apache.org>
Committed: Wed Jun 10 12:40:29 2015 +0530
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../jobhistory/JobHistoryEventHandler.java | 6 ++++-
.../v2/jobhistory/FileNameIndexUtils.java | 17 ++++++++-----
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 ++++++
.../v2/jobhistory/TestFileNameIndexUtils.java | 26 ++++++++++++++++++++
.../src/main/resources/mapred-default.xml | 8 ++++++
6 files changed, 60 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 115fc1b..3c1884d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -465,6 +465,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6388. Remove deprecation warnings from JobHistoryServer classes
(Ray Chiang via ozawa).
+ MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
+ (Siqi Li via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index bf32888..35556a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.security.UserGroupInformation;
@@ -1107,9 +1108,12 @@ public class JobHistoryEventHandler extends AbstractService
if (mi.getHistoryFile() != null) {
Path historyFile = mi.getHistoryFile();
Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile);
+ int jobNameLimit =
+ getConfig().getInt(JHAdminConfig.MR_HS_JOBNAME_LIMIT,
+ JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT);
String doneJobHistoryFileName =
getTempFileName(FileNameIndexUtils.getDoneFileName(mi
- .getJobIndexInfo()));
+ .getJobIndexInfo(), jobNameLimit));
qualifiedDoneFile =
doneDirFS.makeQualified(new Path(doneDirPrefixPath,
doneJobHistoryFileName));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
index 741da11..eb0c54c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
@@ -32,8 +32,6 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId;
public class FileNameIndexUtils {
- static final int JOB_NAME_TRIM_LENGTH = 50;
-
// Sanitize job history file for predictable parsing
static final String DELIMITER = "-";
static final String DELIMITER_ESCAPE = "%2D";
@@ -60,6 +58,12 @@ public class FileNameIndexUtils {
* @return the done job history filename.
*/
public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException {
+ return getDoneFileName(indexInfo,
+ JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT);
+ }
+
+ public static String getDoneFileName(JobIndexInfo indexInfo,
+ int jobNameLimit) throws IOException {
StringBuilder sb = new StringBuilder();
//JobId
sb.append(escapeDelimiters(TypeConverter.fromYarn(indexInfo.getJobId()).toString()));
@@ -74,7 +78,8 @@ public class FileNameIndexUtils {
sb.append(DELIMITER);
//JobName
- sb.append(escapeDelimiters(trimJobName(getJobName(indexInfo))));
+ sb.append(escapeDelimiters(trimJobName(
+ getJobName(indexInfo), jobNameLimit)));
sb.append(DELIMITER);
//FinishTime
@@ -286,9 +291,9 @@ public class FileNameIndexUtils {
/**
* Trims the job-name if required
*/
- private static String trimJobName(String jobName) {
- if (jobName.length() > JOB_NAME_TRIM_LENGTH) {
- jobName = jobName.substring(0, JOB_NAME_TRIM_LENGTH);
+ private static String trimJobName(String jobName, int jobNameLimit) {
+ if (jobName.length() > jobNameLimit) {
+ jobName = jobName.substring(0, jobNameLimit);
}
return jobName;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index f7cba9f..a97c2ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -214,4 +214,11 @@ public class JHAdminConfig {
*/
public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false;
+ /**
+ * Number of characters allowed for job name in Job History Server web page.
+ */
+ public static final String MR_HS_JOBNAME_LIMIT = MR_HISTORY_PREFIX
+ + "jobname.limit";
+ public static final int DEFAULT_MR_HS_JOBNAME_LIMIT = 50;
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
index a0d5fce..3d722e0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
@@ -148,6 +148,32 @@ public class TestFileNameIndexUtils {
}
@Test
+ public void testTrimJobName() throws IOException {
+ int jobNameTrimLength = 5;
+ JobIndexInfo info = new JobIndexInfo();
+ JobID oldJobId = JobID.forName(JOB_ID);
+ JobId jobId = TypeConverter.toYarn(oldJobId);
+ info.setJobId(jobId);
+ info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
+ info.setUser(USER_NAME);
+ info.setJobName(JOB_NAME);
+ info.setFinishTime(Long.parseLong(FINISH_TIME));
+ info.setNumMaps(Integer.parseInt(NUM_MAPS));
+ info.setNumReduces(Integer.parseInt(NUM_REDUCES));
+ info.setJobStatus(JOB_STATUS);
+ info.setQueueName(QUEUE_NAME);
+ info.setJobStartTime(Long.parseLong(JOB_START_TIME));
+
+ String jobHistoryFile =
+ FileNameIndexUtils.getDoneFileName(info, jobNameTrimLength);
+ JobIndexInfo parsedInfo = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
+
+ Assert.assertEquals("Job name did not get trimmed correctly",
+ info.getJobName().substring(0, jobNameTrimLength),
+ parsedInfo.getJobName());
+ }
+
+ @Test
public void testUserNamePercentDecoding() throws IOException {
String jobHistoryFile = String.format(JOB_HISTORY_FILE_FORMATTER,
JOB_ID,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 4a2dd77..3789429 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1706,6 +1706,14 @@
</property>
<property>
+ <name>mapreduce.jobhistory.jobname.limit</name>
+ <value>50</value>
+ <description>
+ Number of characters allowed for job name in Job History Server web page.
+ </description>
+</property>
+
+<property>
<name>mapreduce.job.heap.memory-mb.ratio</name>
<value>0.8</value>
<description>The ratio of heap-size to container-size. If no -Xmx is