You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by zl...@apache.org on 2017/03/20 02:55:24 UTC
svn commit: r1787682 - in /pig/branches/spark/src/org/apache/pig:
backend/hadoop/executionengine/spark/SparkLauncher.java
impl/builtin/StreamingUDF.java scripting/groovy/GroovyEvalFunc.java
Author: zly
Date: Mon Mar 20 02:55:23 2017
New Revision: 1787682
URL: http://svn.apache.org/viewvc?rev=1787682&view=rev
Log:
PIG-5177:Scripting and StreamingPythonUDFs fail with Spark exec type (Adam via Liyun)
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java
pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java
Modified: pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java?rev=1787682&r1=1787681&r2=1787682&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java (original)
+++ pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/SparkLauncher.java Mon Mar 20 02:55:23 2017
@@ -417,6 +417,12 @@ public class SparkLauncher extends Launc
allJars.add(udfJar);
}
+ File scriptUDFJarFile = JarManager.createPigScriptUDFJar(pigContext);
+ if (scriptUDFJarFile != null) {
+ LOG.info("add script udf jar to Spark job");
+ allJars.add(scriptUDFJarFile.getAbsolutePath().toString());
+ }
+
//Upload all jars to spark working directory
for (String jar : allJars) {
File jarFile = new File(jar);
Modified: pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java?rev=1787682&r1=1787681&r2=1787682&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java (original)
+++ pig/branches/spark/src/org/apache/pig/impl/builtin/StreamingUDF.java Mon Mar 20 02:55:23 2017
@@ -242,8 +242,11 @@ public class StreamingUDF extends EvalFu
}
InputStream udfFileStream = this.getClass().getResourceAsStream(
absolutePath + getUserFileExtension());
- command[PATH_TO_FILE_CACHE] = "\"" + userUdfFile.getParentFile().getAbsolutePath()
- + "\"";
+ if (udfFileStream == null) {
+ //Try loading the script from other locally available jars (needed for Spark mode)
+ udfFileStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath+getUserFileExtension());
+ }
+ command[PATH_TO_FILE_CACHE] = "\"" + userUdfFile.getParentFile().getAbsolutePath() + "\"";
try {
FileUtils.copyInputStreamToFile(udfFileStream, userUdfFile);
Modified: pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java
URL: http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java?rev=1787682&r1=1787681&r2=1787682&view=diff
==============================================================================
--- pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java (original)
+++ pig/branches/spark/src/org/apache/pig/scripting/groovy/GroovyEvalFunc.java Mon Mar 20 02:55:23 2017
@@ -74,6 +74,10 @@ public class GroovyEvalFunc<T> extends E
resource = ScriptEngine.class.getResource(File.separator + path);
}
if (resource == null) {
+ //Try loading the script from other locally available jars (needed for Spark mode)
+ resource = Thread.currentThread().getContextClassLoader().getResource(path);
+ }
+ if (resource == null) {
throw new IOException("Cannot find " + path);
}
} else {