You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/08/11 06:17:34 UTC
svn commit: r1617200 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java
test/e2e/pig/tests/nightly.conf
Author: cheolsoo
Date: Mon Aug 11 04:17:34 2014
New Revision: 1617200
URL: http://svn.apache.org/r1617200
Log:
PIG-4112: NPE in packager when union + group-by followed by replicated join in Tez (rohini via cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1617200&r1=1617199&r2=1617200&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Aug 11 04:17:34 2014
@@ -56,6 +56,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4112: NPE in packager when union + group-by followed by replicated join in Tez (rohini via cheolsoo)
+
PIG-4113: TEZ-1386 breaks hadoop 2 compilation in trunk (cheolsoo)
PIG-4110: TEZ-1382 breaks Hadoop 2 compilation (cheolsoo)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1617200&r1=1617199&r2=1617200&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java Mon Aug 11 04:17:34 2014
@@ -137,9 +137,6 @@ public class TezLauncher extends Launche
task.setUncaughtExceptionHandler(jctExceptionHandler);
task.setContextClassLoader(PigContext.getClassLoader());
- // TezJobControl always holds a single TezJob. We use JobControl
- // only because it is convenient to launch the job via
- // ControlledJob.submit().
tezStats.setTezJob(runningJob);
// Mark the times that the jobs were submitted so it's reflected in job
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java?rev=1617200&r1=1617199&r2=1617200&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezPOPackageAnnotator.java Mon Aug 11 04:17:34 2014
@@ -140,7 +140,7 @@ public class TezPOPackageAnnotator exten
@Override
public void visitLocalRearrange(POLocalRearrange lrearrange) throws VisitorException {
POLocalRearrangeTez lr = (POLocalRearrangeTez) lrearrange;
- if (!lr.getOutputKey().equals(pkgTezOp.getOperatorKey().toString())) {
+ if (!(lr.isConnectedToPackage() && lr.getOutputKey().equals(pkgTezOp.getOperatorKey().toString()))) {
return;
}
loRearrangeFound++;
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1617200&r1=1617199&r2=1617200&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Mon Aug 11 04:17:34 2014
@@ -1494,6 +1494,16 @@ d = foreach c generate (name is not NULL
e = filter d by (name matches '.*MIKE.*') OR (NOT (gpa + 1.5 > 4));
store e into ':OUTPATH:';\,
},
+ {
+ 'num' => 13,
+ 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa);
+c = union a, b;
+d = group c by name;
+e = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions);
+f = join d by group, e by name using 'replicated';
+store f into ':OUTPATH:';\,
+ },
]
},
{