You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2012/11/13 16:55:23 UTC
svn commit: r1408803 - in
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec:
HadoopJobExecHelper.java JobDebugger.java
Author: hashutosh
Date: Tue Nov 13 15:55:23 2012
New Revision: 1408803
URL: http://svn.apache.org/viewvc?rev=1408803&view=rev
Log:
HIVE-3215 : JobDebugger should use RunningJob.getTrackingURL (Bhushan Mandhani via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java?rev=1408803&r1=1408802&r2=1408803&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HadoopJobExecHelper.java Tue Nov 13 15:55:23 2012
@@ -20,18 +20,14 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.IOException;
import java.io.Serializable;
-import java.lang.Exception;
-import java.net.MalformedURLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Enumeration;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -40,8 +36,6 @@ import org.apache.hadoop.hive.common.Jav
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.MapRedStats;
import org.apache.hadoop.hive.ql.exec.Operator.ProgressCounter;
-import org.apache.hadoop.hive.ql.exec.errors.ErrorAndSolution;
-import org.apache.hadoop.hive.ql.exec.errors.TaskLogProcessor;
import org.apache.hadoop.hive.ql.history.HiveHistory.Keys;
import org.apache.hadoop.hive.ql.plan.ReducerTimeStatsPerJob;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -470,165 +464,6 @@ public class HadoopJobExecHelper {
}
}
- // Used for showJobFailDebugInfo
- private static class TaskInfo {
- String jobId;
- HashSet<String> logUrls;
-
- public TaskInfo(String jobId) {
- this.jobId = jobId;
- logUrls = new HashSet<String>();
- }
-
- public void addLogUrl(String logUrl) {
- logUrls.add(logUrl);
- }
-
- public HashSet<String> getLogUrls() {
- return logUrls;
- }
-
- public String getJobId() {
- return jobId;
- }
- }
-
- @SuppressWarnings("deprecation")
- private void showJobFailDebugInfo(JobConf conf, RunningJob rj)
- throws IOException, MalformedURLException {
- // Mapping from task ID to the number of failures
- Map<String, Integer> failures = new HashMap<String, Integer>();
- // Successful task ID's
- Set<String> successes = new HashSet<String>();
-
- Map<String, TaskInfo> taskIdToInfo = new HashMap<String, TaskInfo>();
-
- int startIndex = 0;
-
- console.printError("Error during job, obtaining debugging information...");
- // Loop to get all task completion events because getTaskCompletionEvents
- // only returns a subset per call
- while (true) {
- TaskCompletionEvent[] taskCompletions = rj.getTaskCompletionEvents(startIndex);
-
- if (taskCompletions == null || taskCompletions.length == 0) {
- break;
- }
-
- boolean more = true;
- boolean firstError = true;
- for (TaskCompletionEvent t : taskCompletions) {
- // getTaskJobIDs returns Strings for compatibility with Hadoop versions
- // without TaskID or TaskAttemptID
- String[] taskJobIds = ShimLoader.getHadoopShims().getTaskJobIDs(t);
-
- if (taskJobIds == null) {
- console.printError("Task attempt info is unavailable in this Hadoop version");
- more = false;
- break;
- }
-
- // For each task completion event, get the associated task id, job id
- // and the logs
- String taskId = taskJobIds[0];
- String jobId = taskJobIds[1];
- if (firstError) {
- console.printError("Examining task ID: " + taskId + " (and more) from job " + jobId);
- firstError = false;
- }
-
- TaskInfo ti = taskIdToInfo.get(taskId);
- if (ti == null) {
- ti = new TaskInfo(jobId);
- taskIdToInfo.put(taskId, ti);
- }
- // These tasks should have come from the same job.
- assert (ti.getJobId() != null && ti.getJobId().equals(jobId));
- String taskAttemptLogUrl = ShimLoader.getHadoopShims().getTaskAttemptLogUrl(
- conf, t.getTaskTrackerHttp(), t.getTaskId());
- if (taskAttemptLogUrl != null) {
- ti.getLogUrls().add(taskAttemptLogUrl);
- }
-
- // If a task failed, then keep track of the total number of failures
- // for that task (typically, a task gets re-run up to 4 times if it
- // fails
-
- if (t.getTaskStatus() != TaskCompletionEvent.Status.SUCCEEDED) {
- Integer failAttempts = failures.get(taskId);
- if (failAttempts == null) {
- failAttempts = Integer.valueOf(0);
- }
- failAttempts = Integer.valueOf(failAttempts.intValue() + 1);
- failures.put(taskId, failAttempts);
- } else {
- successes.add(taskId);
- }
- }
- if (!more) {
- break;
- }
- startIndex += taskCompletions.length;
- }
- // Remove failures for tasks that succeeded
- for (String task : successes) {
- failures.remove(task);
- }
-
- if (failures.keySet().size() == 0) {
- return;
- }
-
- // Find the highest failure count
- int maxFailures = 0;
- for (Integer failCount : failures.values()) {
- if (maxFailures < failCount.intValue()) {
- maxFailures = failCount.intValue();
- }
- }
-
- // Display Error Message for tasks with the highest failure count
- String jtUrl = JobTrackerURLResolver.getURL(conf);
-
- for (String task : failures.keySet()) {
- if (failures.get(task).intValue() == maxFailures) {
- TaskInfo ti = taskIdToInfo.get(task);
- String jobId = ti.getJobId();
- String taskUrl = jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString();
-
- TaskLogProcessor tlp = new TaskLogProcessor(conf);
- for (String logUrl : ti.getLogUrls()) {
- tlp.addTaskAttemptLogUrl(logUrl);
- }
-
- List<ErrorAndSolution> errors = tlp.getErrors();
-
- StringBuilder sb = new StringBuilder();
- // We use a StringBuilder and then call printError only once as
- // printError will write to both stderr and the error log file. In
- // situations where both the stderr and the log file output is
- // simultaneously output to a single stream, this will look cleaner.
- sb.append("\n");
- sb.append("Task with the most failures(" + maxFailures + "): \n");
- sb.append("-----\n");
- sb.append("Task ID:\n " + task + "\n\n");
- sb.append("URL:\n " + taskUrl + "\n");
-
- for (ErrorAndSolution e : errors) {
- sb.append("\n");
- sb.append("Possible error:\n " + e.getError() + "\n\n");
- sb.append("Solution:\n " + e.getSolution() + "\n");
- }
- sb.append("-----\n");
-
- console.printError(sb.toString());
- // Only print out one task because that's good enough for debugging.
- break;
- }
- }
- return;
-
- }
public void localJobDebugger(int exitVal, String taskId) {
StringBuilder sb = new StringBuilder();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java?rev=1408803&r1=1408802&r2=1408803&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JobDebugger.java Tue Nov 13 15:55:23 2012
@@ -193,10 +193,10 @@ public class JobDebugger implements Runn
// Also keep track of the total number of failures for that
// task (typically, a task gets re-run up to 4 times if it fails.
if (t.getTaskStatus() != TaskCompletionEvent.Status.SUCCEEDED) {
+ String[] diags = rj.getTaskDiagnostics(t.getTaskAttemptId());
+ ti.setDiagnosticMesgs(diags);
if (ti.getErrorCode() == 0) {
- String[] diags = rj.getTaskDiagnostics(t.getTaskAttemptId());
ti.setErrorCode(extractErrorCode(diags));
- ti.setDiagnosticMesgs(diags);
}
Integer failAttempts = failures.get(taskId);
@@ -229,6 +229,10 @@ public class JobDebugger implements Runn
@SuppressWarnings("deprecation")
private void showJobFailDebugInfo() throws IOException {
console.printError("Error during job, obtaining debugging information...");
+ if (!conf.get("mapred.job.tracker", "local").equals("local")) {
+ // Show Tracking URL for remotely running jobs.
+ console.printError("Job Tracking URL: " + rj.getTrackingURL());
+ }
// Loop to get all task completion events because getTaskCompletionEvents
// only returns a subset per call
TaskInfoGrabber tlg = new TaskInfoGrabber();
@@ -265,8 +269,8 @@ public class JobDebugger implements Runn
if (failures.get(task).intValue() == maxFailures) {
TaskInfo ti = taskIdToInfo.get(task);
String jobId = ti.getJobId();
- String taskUrl = (jtUrl == null) ? "Unavailable" :
- jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString();
+ String taskUrl = (jtUrl == null) ? null :
+ jtUrl + "/taskdetails.jsp?jobid=" + jobId + "&tipid=" + task.toString();
TaskLogProcessor tlp = new TaskLogProcessor(conf);
for (String logUrl : ti.getLogUrls()) {
@@ -293,7 +297,9 @@ public class JobDebugger implements Runn
sb.append("Task with the most failures(" + maxFailures + "): \n");
sb.append("-----\n");
sb.append("Task ID:\n " + task + "\n\n");
- sb.append("URL:\n " + taskUrl + "\n");
+ if (taskUrl != null) {
+ sb.append("URL:\n " + taskUrl + "\n");
+ }
for (ErrorAndSolution e : errors) {
sb.append("\n");