You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/04/02 13:48:23 UTC
svn commit: r1583978 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/ src/org/apache/pig/scripting/groovy/
src/org/apache/pig/scripting/jruby/ src/org/apache/pig/scripting/js/
src/org/apache...
Author: cheolsoo
Date: Wed Apr 2 11:48:23 2014
New Revision: 1583978
URL: http://svn.apache.org/r1583978
Log:
PIG-3798: Registered jar in pig script are appended to the classpath multiple times (cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/impl/PigContext.java
pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java
pig/trunk/src/org/apache/pig/scripting/jruby/JrubyScriptEngine.java
pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java
pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1583978&r1=1583977&r2=1583978&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Apr 2 11:48:23 2014
@@ -99,6 +99,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3798: Registered jar in pig script are appended to the classpath multiple times (cheolsoo)
+
PIG-3844: Make ScriptState InheritableThreadLocal for threads that need it (amatsukawa via cheolsoo)
PIG-3837: ant pigperf target is broken in trunk (cheolsoo)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1583978&r1=1583977&r2=1583978&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Wed Apr 2 11:48:23 2014
@@ -1609,7 +1609,7 @@ public class JobControlCompiler{
Path pathInHDFS = shipToHDFS(pigContext, conf, url);
// and add to the DistributedCache
DistributedCache.addFileToClassPath(pathInHDFS, conf);
- pigContext.skipJars.add(url.getPath());
+ pigContext.addSkipJar(url.getPath());
}
private static Path getCacheStagingDir(Configuration conf) throws IOException {
Modified: pig/trunk/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=1583978&r1=1583977&r2=1583978&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/PigContext.java Wed Apr 2 11:48:23 2014
@@ -112,7 +112,7 @@ public class PigContext implements Seria
transient private Map<URL, String> extraJarOriginalPaths = new HashMap<URL, String>();
// jars needed for scripting udfs - jython.jar etc
- public List<String> scriptJars = new ArrayList<String>(2);
+ transient public List<String> scriptJars = new ArrayList<String>(2);
// jars that should not be merged in.
// (some functions may come from pig.jar and we don't want the whole jar file.)
@@ -120,7 +120,7 @@ public class PigContext implements Seria
// jars that are predeployed to the cluster and thus should not be merged in at all (even subsets).
transient public Vector<String> predeployedJars = new Vector<String>(2);
-
+
// script files that are needed to run a job
@Deprecated
public List<String> scriptFiles = new ArrayList<String>();
@@ -259,9 +259,9 @@ public class PigContext implements Seria
String pigJar = JarManager.findContainingJar(Main.class);
String hadoopJar = JarManager.findContainingJar(FileSystem.class);
if (pigJar != null) {
- skipJars.add(pigJar);
+ addSkipJar(pigJar);
if (!pigJar.equals(hadoopJar))
- skipJars.add(hadoopJar);
+ addSkipJar(hadoopJar);
}
this.executionEngine = execType.getExecutionEngine(this);
@@ -320,18 +320,7 @@ public class PigContext implements Seria
* @param path
*/
public void addScriptFile(String path) {
- if (path != null) {
- aliasedScriptFiles.put(path.replaceFirst("^/", "").replaceAll(":", ""), new File(path));
- }
- }
-
- public boolean hasJar(String path) {
- for (URL url : extraJars) {
- if (extraJarOriginalPaths.get(url).equals(path)) {
- return true;
- }
- }
- return false;
+ addScriptFile(path, path);
}
/**
@@ -346,6 +335,18 @@ public class PigContext implements Seria
}
}
+ public void addScriptJar(String path) {
+ if (path != null && !scriptJars.contains(path)) {
+ scriptJars.add(path);
+ }
+ }
+
+ public void addSkipJar(String path) {
+ if (path != null && !skipJars.contains(path)) {
+ skipJars.add(path);
+ }
+ }
+
public void addJar(String path) throws MalformedURLException {
if (path != null) {
URL resource = (new File(path)).toURI().toURL();
@@ -354,14 +355,23 @@ public class PigContext implements Seria
}
public void addJar(URL resource, String originalPath) throws MalformedURLException{
- if (resource != null) {
+ if (resource != null && !extraJars.contains(resource)) {
extraJars.add(resource);
extraJarOriginalPaths.put(resource, originalPath);
classloader.addURL(resource);
Thread.currentThread().setContextClassLoader(PigContext.classloader);
}
}
-
+
+ public boolean hasJar(String path) {
+ for (URL url : extraJars) {
+ if (extraJarOriginalPaths.get(url).equals(path)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Adds the specified path to the predeployed jars list. These jars will
* never be included in generated job jar.
@@ -370,7 +380,9 @@ public class PigContext implements Seria
* cluster to reduce the size of the job jar.
*/
public void markJarAsPredeployed(String path) {
- predeployedJars.add(path);
+ if (path != null && !predeployedJars.contains(path)) {
+ predeployedJars.add(path);
+ }
}
public String doParamSubstitution(InputStream in,
Modified: pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java?rev=1583978&r1=1583977&r2=1583978&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/groovy/GroovyScriptEngine.java Wed Apr 2 11:48:23 2014
@@ -136,7 +136,7 @@ public class GroovyScriptEngine extends
public void registerFunctions(String path, String namespace, PigContext pigContext) throws IOException {
if (!isInitialized) {
- pigContext.scriptJars.add(getJarPath(groovy.util.GroovyScriptEngine.class));
+ pigContext.addScriptJar(getJarPath(groovy.util.GroovyScriptEngine.class));
isInitialized = true;
}
Modified: pig/trunk/src/org/apache/pig/scripting/jruby/JrubyScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/jruby/JrubyScriptEngine.java?rev=1583978&r1=1583977&r2=1583978&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/jruby/JrubyScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/jruby/JrubyScriptEngine.java Wed Apr 2 11:48:23 2014
@@ -129,7 +129,7 @@ public class JrubyScriptEngine extends S
@Override
public void registerFunctions(String path, String namespace, PigContext pigContext) throws IOException {
if (!isInitialized) {
- pigContext.scriptJars.add(getJarPath(Ruby.class));
+ pigContext.addScriptJar(getJarPath(Ruby.class));
pigContext.addScriptFile("pigudf.rb", "pigudf.rb");
isInitialized = true;
}
@@ -137,8 +137,6 @@ public class JrubyScriptEngine extends S
for (Map.Entry<String,Object> entry : RubyFunctions.getFunctions("evalfunc", path).entrySet()) {
String method = entry.getKey();
- String functionType = rubyEngine.callMethod(entry.getValue(), "name", String.class);
-
FuncSpec funcspec = new FuncSpec(JrubyEvalFunc.class.getCanonicalName() + "('" + path + "','" + method +"')");
pigContext.registerFunction(namespace + "." + method, funcspec);
}
@@ -197,14 +195,14 @@ public class JrubyScriptEngine extends S
if (file.isDirectory()) {
continue;
} else if (file.getName().endsWith(".jar") || file.getName().endsWith(".zip")) {
- pigContext.scriptJars.add(file.getPath());
+ pigContext.addScriptJar(file.getPath());
} else {
String localPath = libFile.getName() + file.getPath().replaceFirst(libFile.getPath(), "");
pigContext.addScriptFile(localPath, file.getPath());
}
}
} else {
- pigContext.scriptJars.add(lib);
+ pigContext.addScriptJar(lib);
}
}
}
Modified: pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java?rev=1583978&r1=1583977&r2=1583978&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java Wed Apr 2 11:48:23 2014
@@ -230,9 +230,9 @@ public class JsScriptEngine extends Scri
PigContext pigContext) throws IOException {
// to enable passing of information to the slave
this.scriptPath = path;
- this.clientInstance = this;
-
- pigContext.scriptJars.add(getJarPath(Context.class));
+ JsScriptEngine.clientInstance = this;
+
+ pigContext.addScriptJar(getJarPath(Context.class));
namespace = (namespace == null) ? "" : namespace + NAMESPACE_SEPARATOR;
FileInputStream fis = new FileInputStream(path);
try {
Modified: pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java?rev=1583978&r1=1583977&r2=1583978&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java Wed Apr 2 11:48:23 2014
@@ -228,7 +228,7 @@ public class JythonScriptEngine extends
if (modulepath.equals(JVM_JAR)) {
continue;
} else if (modulepath.endsWith(".jar") || modulepath.endsWith(".zip")) {
- pigContext.scriptJars.add(modulepath);
+ pigContext.addScriptJar(modulepath);
} else {
pigContext.addScriptFile(modulename, modulepath);
}
@@ -342,7 +342,7 @@ public class JythonScriptEngine extends
throws IOException {
Interpreter.setMain(false);
Interpreter.init(path, pigContext);
- pigContext.scriptJars.add(getJarPath(PythonInterpreter.class));
+ pigContext.addScriptJar(getJarPath(PythonInterpreter.class));
PythonInterpreter pi = Interpreter.interpreter;
@SuppressWarnings("unchecked")
List<PyTuple> locals = ((PyStringMap) pi.getLocals()).items();