You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2008/09/29 10:08:50 UTC
svn commit: r700021 - in /hadoop/core/trunk: CHANGES.txt
conf/hadoop-default.xml
src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
src/mapred/org/apache/hadoop/mapred/JobHistory.java
src/webapps/job/loadhistory.jsp
Author: ddas
Date: Mon Sep 29 01:08:50 2008
New Revision: 700021
URL: http://svn.apache.org/viewvc?rev=700021&view=rev
Log:
HADOOP-4189. Fixes the history blocksize & intertracker protocol version issues introduced as part of HADOOP-3245. Contributed by Amar Kamat.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/conf/hadoop-default.xml
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/core/trunk/src/webapps/job/loadhistory.jsp
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Sep 29 01:08:50 2008
@@ -776,6 +776,9 @@
memory management in TaskTracker are disabled on Windows.
(Vinod K V via rangadi)
+ HADOOP-4189. Fixes the history blocksize & intertracker protocol version
+ issues introduced as part of HADOOP-3245. (Amar Kamat via ddas)
+
Release 0.18.2 - Unreleased
BUG FIXES
Modified: hadoop/core/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/hadoop-default.xml?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/conf/hadoop-default.xml (original)
+++ hadoop/core/trunk/conf/hadoop-default.xml Mon Sep 29 01:08:50 2008
@@ -762,19 +762,11 @@
<property>
<name>mapred.jobtracker.job.history.block.size</name>
- <value>0</value>
+ <value>3145728></value>
<description>The block size of the job history file. Since the job recovery
uses job history, its important to dump job history to disk as
- soon as possible.
- </description>
-</property>
-
-<property>
- <name>mapred.jobtracker.job.history.buffer.size</name>
- <value>4096</value>
- <description>The buffer size for the job history file. Since the job
- recovery uses job history, its important to frequently flush the
- job history to disk. This will minimize the loss in recovery.
+ soon as possible. Note that this is an expert level parameter.
+ The default value is set to 3 MB.
</description>
</property>
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/InterTrackerProtocol.java Mon Sep 29 01:08:50 2008
@@ -48,6 +48,8 @@
* Version 16: adds ResourceStatus to TaskTrackerStatus for HADOOP-3759
* Version 17: Changed format of Task and TaskStatus for HADOOP-3150
* Version 18: Changed status message due to changes in TaskStatus
+ * Changed heartbeat to piggyback JobTracker restart information
+ so that the TaskTracker can synchronize itself.
*/
public static final long versionID = 18L;
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java Mon Sep 29 01:08:50 2008
@@ -85,7 +85,6 @@
private static boolean disableHistory = false;
private static final String SECONDARY_FILE_SUFFIX = ".recover";
private static long jobHistoryBlockSize = 0;
- private static int jobHistoryBufferSize = 0;
private static String jobtrackerHostname;
/**
* Record types are identifiers for each line of log in history files.
@@ -148,14 +147,10 @@
}
conf.set("hadoop.job.history.location", LOG_DIR);
disableHistory = false;
- // set the job history block size
+ // set the job history block size (default is 3MB)
jobHistoryBlockSize =
conf.getLong("mapred.jobtracker.job.history.block.size",
- fs.getDefaultBlockSize());
- // set the job history buffer size
- jobHistoryBufferSize =
- Integer.parseInt(conf.get("mapred.jobtracker.job.history.buffer.size",
- "4096"));
+ 3 * 1024 * 1024);
} catch(IOException e) {
LOG.error("Failed to initialize JobHistory log file", e);
disableHistory = true;
@@ -729,8 +724,10 @@
logFile = recoverJobHistoryFile(jobConf, logFile);
+ int defaultBufferSize =
+ fs.getConf().getInt("io.file.buffer.size", 4096);
out = fs.create(logFile, FsPermission.getDefault(), true,
- jobHistoryBufferSize,
+ defaultBufferSize,
fs.getDefaultReplication(),
jobHistoryBlockSize, null);
writer = new PrintWriter(out);
Modified: hadoop/core/trunk/src/webapps/job/loadhistory.jsp
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/webapps/job/loadhistory.jsp?rev=700021&r1=700020&r2=700021&view=diff
==============================================================================
--- hadoop/core/trunk/src/webapps/job/loadhistory.jsp (original)
+++ hadoop/core/trunk/src/webapps/job/loadhistory.jsp Mon Sep 29 01:08:50 2008
@@ -24,9 +24,16 @@
// then remove the attribute
// if the job has not yet finished, remove the attribute sothat it
// gets refreshed.
+ boolean isJobComplete = false;
+ if (null != job) {
+ String jobStatus = job.get(Keys.JOB_STATUS);
+ isJobComplete = Values.SUCCESS.name() == jobStatus
+ || Values.FAILED.name() == jobStatus
+ || Values.KILLED.name() == jobStatus;
+ }
if (null != job &&
(!jobId.equals(job.get(Keys.JOBID))
- || job.get(Keys.JOB_STATUS) == "")) {
+ || !isJobComplete)) {
// remove jobInfo from session, keep only one job in session at a time
request.getSession().removeAttribute("job");
job = null ;