You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by zl...@apache.org on 2017/04/19 08:47:27 UTC
svn commit: r1791880 -
/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Author: zly
Date: Wed Apr 19 08:47:27 2017
New Revision: 1791880
URL: http://svn.apache.org/viewvc?rev=1791880&view=rev
Log:
PIG-5176:Several ComputeSpec test cases fail-2(Nandor via Liyun)
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1791880&r1=1791879&r2=1791880&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Wed Apr 19 08:47:27 2017
@@ -557,6 +557,12 @@ public class SparkLauncher extends Launc
sparkConf.setMaster(master);
sparkConf.setAppName(pigCtxtProperties.getProperty(PigContext.JOB_NAME,"pig"));
+ // On Spark 1.6, Netty file server doesn't allow adding the same file with the same name twice
+ // This is a problem for streaming using a script + explicit ship the same script combination (PIG-5134)
+ // HTTP file server doesn't have this restriction, it overwrites the file if added twice
+ String useNettyFileServer = pigCtxtProperties.getProperty(PigConfiguration.PIG_SPARK_USE_NETTY_FILESERVER, "false");
+ sparkConf.set("spark.rpc.useNettyFileServer", useNettyFileServer);
+
if (sparkHome != null && !sparkHome.isEmpty()) {
sparkConf.setSparkHome(sparkHome);
} else {