You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/09/25 01:50:29 UTC
svn commit: r1526053 - in /pig/trunk: ./
shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/
shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/
src/org/apache/pig/backend/hadoop/executionengine/
Author: daijy
Date: Tue Sep 24 23:50:29 2013
New Revision: 1526053
URL: http://svn.apache.org/r1526053
Log:
PIG-3360: Some intermittent negative e2e tests fail on hadoop 2
Modified:
pig/trunk/CHANGES.txt
pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/Launcher.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1526053&r1=1526052&r2=1526053&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Sep 24 23:50:29 2013
@@ -251,6 +251,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3360: Some intermittent negative e2e tests fail on hadoop 2 (daijy)
+
PIG-3468: PIG-3123 breaks e2e test Jython_Diagnostics_2 (daijy)
PIG-3471: Add a base abstract class for ExecutionEngine (cheolsoo)
Modified: pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1526053&r1=1526052&r2=1526053&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/trunk/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Tue Sep 24 23:50:29 2013
@@ -26,6 +26,7 @@ import org.apache.hadoop.mapred.Counters
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapred.jobcontrol.JobControl;
+import org.apache.hadoop.mapred.TaskReport;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -105,4 +106,13 @@ public class HadoopShims {
JobClient jobClient = job.getJobClient();
return jobClient.getJob(job.getAssignedJobID()).getCounters();
}
+
+ public static boolean isJobFailed(TaskReport report) {
+ float successfulProgress = 1.0f;
+ // if the progress reported is not 1.0f then the map or reduce
+ // job failed
+ // this comparison is in place for the backward compatibility
+ // for Hadoop 0.20
+ return report.getProgress() != successfulProgress;
+ }
}
Modified: pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1526053&r1=1526052&r2=1526053&view=diff
==============================================================================
--- pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/trunk/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Tue Sep 24 23:50:29 2013
@@ -24,6 +24,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TIPStatus;
+import org.apache.hadoop.mapred.TaskReport;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapred.jobcontrol.JobControl;
import org.apache.hadoop.mapreduce.ContextFactory;
@@ -109,4 +111,8 @@ public class HadoopShims {
public static Counters getCounters(Job job) throws IOException, InterruptedException {
return new Counters(job.getJob().getCounters());
}
+
+ public static boolean isJobFailed(TaskReport report) {
+ return report.getCurrentStatus()==TIPStatus.FAILED;
+ }
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/Launcher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/Launcher.java?rev=1526053&r1=1526052&r2=1526053&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/Launcher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/Launcher.java Tue Sep 24 23:50:29 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapred.JobClien
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.TIPStatus;
import org.apache.hadoop.mapred.TaskReport;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapred.jobcontrol.JobControl;
@@ -42,6 +43,7 @@ import org.apache.pig.backend.BackendExc
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobCreationException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.impl.plan.VisitorException;
@@ -198,14 +200,8 @@ public abstract class Launcher {
ArrayList<Exception> exceptions = new ArrayList<Exception>();
String exceptionCreateFailMsg = null;
boolean jobFailed = false;
- float successfulProgress = 1.0f;
if (msgs.length > 0) {
- // if the progress reported is not 1.0f then the map or reduce
- // job failed
- // this comparison is in place till Hadoop 0.20 provides methods
- // to query
- // job status
- if (reports[i].getProgress() != successfulProgress) {
+ if (HadoopShims.isJobFailed(reports[i])) {
jobFailed = true;
}
Set<String> errorMessageSet = new HashSet<String>();
@@ -234,7 +230,7 @@ public abstract class Launcher {
}
}
// if there are no valid exception that could be created, report
- if ((exceptions.size() == 0) && (exceptionCreateFailMsg != null)) {
+ if (jobFailed && (exceptions.size() == 0) && (exceptionCreateFailMsg != null)) {
int errCode = 2997;
String msg = "Unable to recreate exception from backed error: "
+ exceptionCreateFailMsg;