You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by sz...@apache.org on 2017/07/22 11:28:03 UTC
svn commit: r1802675 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/spark/
src/org/apache/pig/backend/hadoop/executionengine/spark/converter/
Author: szita
Date: Sat Jul 22 11:28:02 2017
New Revision: 1802675
URL: http://svn.apache.org/viewvc?rev=1802675&view=rev
Log:
PIG-5274: TestEvalPipelineLocal#testSetLocationCalledInFE is failing in spark mode after PIG-5157 (nkollar via szita)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1802675&r1=1802674&r2=1802675&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Jul 22 11:28:02 2017
@@ -38,6 +38,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5274: TestEvalPipelineLocal#testSetLocationCalledInFE is failing in spark mode after PIG-5157 (nkollar via szita)
+
PIG-4767: Partition filter not pushed down when filter clause references variable from another load path (knoguchi)
PIG-5270: Typo in Pig Logging (FromAlaska49 via daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1802675&r1=1802674&r2=1802675&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Sat Jul 22 11:28:02 2017
@@ -175,7 +175,6 @@ public class SparkLauncher extends Launc
SparkPigStats sparkStats = (SparkPigStats) pigContext
.getExecutionEngine().instantiatePigStats();
sparkStats.initialize(pigContext, sparkplan, jobConf);
- UDFContext.getUDFContext().addJobConf(jobConf);
PigStats.start(sparkStats);
startSparkIfNeeded(jobConf, pigContext);
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java?rev=1802675&r1=1802674&r2=1802675&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkShims.java Sat Jul 22 11:28:02 2017
@@ -60,7 +60,12 @@ public abstract class SparkShims impleme
public static SparkShims getInstance() {
if (sparkShims == null) {
- String sparkVersion = UDFContext.getUDFContext().getJobConf().get(SPARK_VERSION, "");
+ String sparkVersion;
+ if (UDFContext.getUDFContext().isFrontend()) {
+ sparkVersion = SparkContext.getOrCreate().version();
+ } else {
+ sparkVersion = UDFContext.getUDFContext().getJobConf().get(SPARK_VERSION, "");
+ }
LOG.info("Initializing SparkShims for Spark version: " + sparkVersion);
String sparkMajorVersion = getSparkMajorVersion(sparkVersion);
try {
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java?rev=1802675&r1=1802674&r2=1802675&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/spark/converter/SkewedJoinConverter.java Sat Jul 22 11:28:02 2017
@@ -232,8 +232,8 @@ public class SkewedJoinConverter impleme
}
return result;
- } catch (Exception e) {
- log.warn(e);
+ } catch (ExecException e) {
+ log.error(e);
return null;
}
}