You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2016/04/12 21:32:09 UTC
svn commit: r1738858 - in /pig/branches/branch-0.15: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
test/org/apache/pig/test/TestGrunt.java
Author: rohini
Date: Tue Apr 12 19:32:09 2016
New Revision: 1738858
URL: http://svn.apache.org/viewvc?rev=1738858&view=rev
Log:
PIG-4867: -stop_on_failure does not work with Tez (rohini)
Modified:
pig/branches/branch-0.15/CHANGES.txt
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
pig/branches/branch-0.15/test/org/apache/pig/test/TestGrunt.java
Modified: pig/branches/branch-0.15/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/CHANGES.txt?rev=1738858&r1=1738857&r2=1738858&view=diff
==============================================================================
--- pig/branches/branch-0.15/CHANGES.txt (original)
+++ pig/branches/branch-0.15/CHANGES.txt Tue Apr 12 19:32:09 2016
@@ -28,6 +28,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4867: -stop_on_failure does not work with Tez (rohini)
+
PIG-4851: Null not padded when input has less fields than declared schema for some loader (rohini)
PIG-4812: Register Groovy UDF with relative path does not work (daijy)
Modified: pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1738858&r1=1738857&r2=1738858&view=diff
==============================================================================
--- pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java (original)
+++ pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java Tue Apr 12 19:32:09 2016
@@ -168,6 +168,10 @@ public class TezLauncher extends Launche
tezScriptState.emitInitialPlanNotification(tezPlanContainer);
tezScriptState.emitLaunchStartedNotification(tezPlanContainer.size()); //number of DAGs to Launch
+ boolean stop_on_failure =
+ Boolean.valueOf(pc.getProperties().getProperty("stop.on.failure", "false"));
+ boolean stoppedOnFailure = false;
+
TezPlanContainerNode tezPlanContainerNode;
TezOperPlan tezPlan;
int processedDAGs = 0;
@@ -246,7 +250,18 @@ public class TezLauncher extends Launche
((tezPlanContainer.size() - processedDAGs)/tezPlanContainer.size()) * 100);
}
handleUnCaughtException(pc);
- tezPlanContainer.updatePlan(tezPlan, reporter.notifyFinishedOrFailed());
+ boolean tezDAGSucceeded = reporter.notifyFinishedOrFailed();
+ tezPlanContainer.updatePlan(tezPlan, tezDAGSucceeded);
+ // if stop_on_failure is enabled, we need to stop immediately when any job has failed
+ if (!tezDAGSucceeded) {
+ if (stop_on_failure) {
+ stoppedOnFailure = true;
+ break;
+ } else {
+ log.warn("Ooops! Some job has failed! Specify -stop_on_failure if you "
+ + "want Pig to stop immediately on failure.");
+ }
+ }
}
tezStats.finish();
@@ -273,6 +288,11 @@ public class TezLauncher extends Launche
}
}
+ if (stoppedOnFailure) {
+ throw new ExecException("Stopping execution on job failure with -stop_on_failure option", 6017,
+ PigException.REMOTE_ENVIRONMENT);
+ }
+
return tezStats;
}
Modified: pig/branches/branch-0.15/test/org/apache/pig/test/TestGrunt.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.15/test/org/apache/pig/test/TestGrunt.java?rev=1738858&r1=1738857&r2=1738858&view=diff
==============================================================================
--- pig/branches/branch-0.15/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/branches/branch-0.15/test/org/apache/pig/test/TestGrunt.java Tue Apr 12 19:32:09 2016
@@ -963,7 +963,6 @@ public class TestGrunt {
@Test
public void testStopOnFailure() throws Throwable {
- Assume.assumeTrue("Skip this test for TEZ", Util.isMapredExecType(cluster.getExecType()));
PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
PigContext context = server.getPigContext();
context.getProperties().setProperty("stop.on.failure", ""+true);
@@ -1473,7 +1472,7 @@ public class TestGrunt {
JavaCompilerHelper javaCompilerHelper = new JavaCompilerHelper();
javaCompilerHelper.compile(tmpDir.getAbsolutePath(),
new JavaCompilerHelper.JavaSourceFromString("com.xxx.udf.TestUDF", udfSrc));
-
+
String jarName = "TestUDFJar.jar";
String jarFile = tmpDir.getAbsolutePath() + FILE_SEPARATOR + jarName;
int status = Util.executeJavaCommand("jar -cf " + jarFile +