You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by zl...@apache.org on 2017/03/12 21:13:50 UTC
svn commit: r1786613 -
/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/JobGraphBuilder.java
Author: zly
Date: Sun Mar 12 21:13:49 2017
New Revision: 1786613
URL: http://svn.apache.org/viewvc?rev=1786613&view=rev
Log:
PIG-5180: MergeSparseJoin fails with Spark exec type (Adam via Liyun)
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/JobGraphBuilder.java
Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/JobGraphBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/JobGraphBuilder.java?rev=1786613&r1=1786612&r2=1786613&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/JobGraphBuilder.java (original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/JobGraphBuilder.java Sun Mar 12 21:13:49 2017
@@ -59,6 +59,7 @@ import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.newplan.logical.relational.LOJoin;
import org.apache.pig.tools.pigstats.spark.SparkPigStats;
import org.apache.pig.tools.pigstats.spark.SparkStatsUtil;
import org.apache.spark.api.java.JavaSparkContext;
@@ -121,7 +122,9 @@ public class JobGraphBuilder extends Spa
if (poMergeJoins.size() > 0) {
for (POMergeJoin poMergeJoin : poMergeJoins) {
String idxFileName = poMergeJoin.getIndexFile();
- filesForMoreReplication.add(new Path(idxFileName));
+ if (idxFileName != null) {
+ filesForMoreReplication.add(new Path(idxFileName));
+ }
// in spark mode, set as null so that PoMergeJoin won't use hadoop distributed cache
// see POMergeJoin.seekInRightStream()
poMergeJoin.setIndexFile(null);