You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by am...@apache.org on 2010/06/29 11:36:13 UTC
svn commit: r958902 - in /hadoop/mapreduce/trunk: ./
src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/
src/test/mapred/org/apache/hadoop/mapred/ src/webapps/job/
Author: amareshwari
Date: Tue Jun 29 09:36:12 2010
New Revision: 958902
URL: http://svn.apache.org/viewvc?rev=958902&view=rev
Log:
MAPREDUCE-1850. Includes job submit host information (name and ip) in jobconf and jobdetails display. Contributed by Krishna Ramachandran
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobSubmitter.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java
hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=958902&r1=958901&r2=958902&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Jun 29 09:36:12 2010
@@ -67,6 +67,9 @@ Trunk (unreleased changes)
MAPREDUCE-1788. Ensure failure to setup CompletedJobStatusStore is not
silently ignored by the JobTracker. (Krishna Ramachandran via acmurthy)
+ MAPREDUCE-1850. Includes job submit host information (name and ip) in
+ jobconf and jobdetails display (Krishna Ramachandran via amareshwari)
+
OPTIMIZATIONS
MAPREDUCE-1354. Enhancements to JobTracker for better performance and
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=958902&r1=958901&r2=958902&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Tue Jun 29 09:36:12 2010
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapred.lib.Hash
import org.apache.hadoop.mapred.lib.KeyFieldBasedComparator;
import org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner;
import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.util.ConfigUtil;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
@@ -1438,6 +1439,46 @@ public class JobConf extends Configurati
}
/**
+ * Set JobSubmitHostName for this job.
+ *
+ * @param hostname the JobSubmitHostName for this job.
+ */
+ void setJobSubmitHostName(String hostname) {
+ set(MRJobConfig.JOB_SUBMITHOST, hostname);
+ }
+
+ /**
+ * Get the JobSubmitHostName for this job.
+ *
+ * @return the JobSubmitHostName for this job.
+ */
+ String getJobSubmitHostName() {
+ String hostname = get(MRJobConfig.JOB_SUBMITHOST);
+
+ return hostname;
+ }
+
+ /**
+ * Set JobSubmitHostAddress for this job.
+ *
+ * @param hostadd the JobSubmitHostAddress for this job.
+ */
+ void setJobSubmitHostAddress(String hostadd) {
+ set(MRJobConfig.JOB_SUBMITHOSTADDR, hostadd);
+ }
+
+ /**
+ * Get JobSubmitHostAddress for this job.
+ *
+ * @return JobSubmitHostAddress for this job.
+ */
+ String getJobSubmitHostAddress() {
+ String hostadd = get(MRJobConfig.JOB_SUBMITHOSTADDR);
+
+ return hostadd;
+ }
+
+ /**
* Get whether the task profiling is enabled.
* @return true if some tasks will be profiled
*/
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=958902&r1=958901&r2=958902&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Tue Jun 29 09:36:12 2010
@@ -261,6 +261,8 @@ public class JobInProgress {
new TreeMap<TaskAttemptID, Integer>();
private Object schedulingInfo;
+ private String submitHostName;
+ private String submitHostAddress;
//thresholds for speculative execution
float slowTaskThreshold;
@@ -472,6 +474,8 @@ public class JobInProgress {
this.nonRunningReduces = new LinkedList<TaskInProgress>();
this.runningReduces = new LinkedHashSet<TaskInProgress>();
this.resourceEstimator = new ResourceEstimator(this);
+ this.submitHostName = conf.getJobSubmitHostName();
+ this.submitHostAddress = conf.getJobSubmitHostAddress();
this.nonLocalMaps = new LinkedList<TaskInProgress>();
this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>();
@@ -3702,4 +3706,11 @@ public class JobInProgress {
+ keysFile.toUri().getPath());
}
+ public String getJobSubmitHostAddress() {
+ return submitHostAddress;
+ }
+
+ public String getJobSubmitHostName() {
+ return submitHostName;
+ }
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobSubmitter.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobSubmitter.java?rev=958902&r1=958901&r2=958902&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobSubmitter.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/JobSubmitter.java Tue Jun 29 09:36:12 2010
@@ -56,6 +56,8 @@ class JobSubmitter {
protected static final Log LOG = LogFactory.getLog(JobSubmitter.class);
private FileSystem jtFs;
private ClientProtocol submitClient;
+ private String submitHostName;
+ private String submitHostAddress;
JobSubmitter(FileSystem submitFs, ClientProtocol submitClient)
throws IOException {
@@ -317,6 +319,13 @@ class JobSubmitter {
job.getConfiguration());
//configure the command line options correctly on the submitting dfs
Configuration conf = job.getConfiguration();
+ InetAddress ip = InetAddress.getLocalHost();
+ if (ip != null) {
+ submitHostAddress = ip.getHostAddress();
+ submitHostName = ip.getHostName();
+ conf.set(MRJobConfig.JOB_SUBMITHOST,submitHostName);
+ conf.set(MRJobConfig.JOB_SUBMITHOSTADDR,submitHostAddress);
+ }
JobID jobId = submitClient.getNewJobID();
job.setJobID(jobId);
Path submitJobDir = new Path(jobStagingArea, jobId.toString());
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=958902&r1=958901&r2=958902&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/MRJobConfig.java Tue Jun 29 09:36:12 2010
@@ -267,5 +267,8 @@ public interface MRJobConfig {
public static final String JOB_ACL_VIEW_JOB = "mapreduce.job.acl-view-job";
public static final String JOB_ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job";
-
-}
\ No newline at end of file
+ public static final String JOB_SUBMITHOST =
+ "mapreduce.job.submithostname";
+ public static final String JOB_SUBMITHOSTADDR =
+ "mapreduce.job.submithostaddress";
+}
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java?rev=958902&r1=958901&r2=958902&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestEmptyJob.java Tue Jun 29 09:36:12 2010
@@ -21,6 +21,7 @@ package org.apache.hadoop.mapred;
import java.io.File;
import java.io.IOException;
import java.net.URI;
+import java.net.InetAddress;
import junit.framework.TestCase;
@@ -119,6 +120,13 @@ public class TestEmptyJob extends TestCa
RunningJob runningJob = jc.submitJob(conf);
JobInProgress job = mr.getJobTrackerRunner().getJobTracker().getJob(runningJob.getID());
+ InetAddress ip = InetAddress.getLocalHost();
+ if (ip != null) {
+ assertTrue(job.getJobSubmitHostAddress().equalsIgnoreCase(
+ ip.getHostAddress()));
+ assertTrue(job.getJobSubmitHostName().equalsIgnoreCase(ip.getHostName()));
+ }
+
while (true) {
if (job.isCleanupLaunched()) {
LOG.info("Waiting for cleanup to be launched for job "
Modified: hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp?rev=958902&r1=958901&r2=958902&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp Tue Jun 29 09:36:12 2010
@@ -304,6 +304,10 @@
HtmlQuoting.quoteHtmlChars(profile.getJobName()) + "<br>\n");
out.print("<b>Job File:</b> <a href=\"jobconf.jsp?jobid=" + jobId + "\">" +
profile.getJobFile() + "</a><br>\n");
+ out.print("<b>Submit Host:</b> " +
+ HtmlQuoting.quoteHtmlChars(job.getJobSubmitHostName()) + "<br>\n");
+ out.print("<b>Submit Host Address:</b> " +
+ HtmlQuoting.quoteHtmlChars(job.getJobSubmitHostAddress()) + "<br>\n");
Map<JobACL, AccessControlList> jobAcls = status.getJobACLs();
JSPUtil.printJobACLs(tracker, jobAcls, out);