You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/08/07 19:47:09 UTC
svn commit: r1370393 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/PigServer.java src/org/apache/pig/impl/PigContext.java
src/org/apache/pig/impl/util/JarManager.java
src/org/apache/pig/scripting/ScriptEngine.java
Author: daijy
Date: Tue Aug 7 17:47:08 2012
New Revision: 1370393
URL: http://svn.apache.org/viewvc?rev=1370393&view=rev
Log:
PIG-2761: With hadoop23 importing modules inside python script does not work
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java
pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java
pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Tue Aug 7 17:47:08 2012
@@ -30,6 +30,8 @@ PIG-2619: HBaseStorage constructs a Scan
BUG FIXES
+PIG-2761: With hadoop23 importing modules inside python script does not work (rohini via daijy)
+
PIG-2775: Register jar does not goes to classpath in some cases (daijy)
PIG-2666: LoadFunc.setLocation() is not called when pig script only has Order By (daijy)
Modified: pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/PigServer.java?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/PigServer.java Tue Aug 7 17:47:08 2012
@@ -563,11 +563,16 @@ public class PigServer {
throw new FrontendException(msg, errCode,
PigException.USER_ENVIRONMENT);
}
+ String cwd = new File(".").getCanonicalPath();
+ String filePath = f.getCanonicalPath();
+ //Use the relative path in the jar, if the path specified is relative
+ String nameInJar = filePath.equals(cwd + File.separator + path) ?
+ filePath.substring(cwd.length() + 1) : filePath;
+ pigContext.addScriptFile(nameInJar, filePath);
if(scriptingLang != null) {
ScriptEngine se = ScriptEngine.getInstance(scriptingLang);
- se.registerFunctions(path, namespace, pigContext);
+ se.registerFunctions(nameInJar, namespace, pigContext);
}
- pigContext.addScriptFile(path);
}
/**
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/PigContext.java Tue Aug 7 17:47:08 2012
@@ -28,6 +28,7 @@ import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -91,7 +92,9 @@ public class PigContext implements Seria
private Properties properties;
// script files that are needed to run a job
+ @Deprecated
public List<String> scriptFiles = new ArrayList<String>();
+ private Map<String,File> aliasedScriptFiles = new LinkedHashMap<String,File>();
// script jars that are needed to run a script - jython.jar etc
public List<String> scriptJars = new ArrayList<String>(2);
@@ -212,13 +215,31 @@ public class PigContext implements Seria
log.error("Failed to set tracker at: " + newLocation);
}
}
-
+
+ /**
+ * calls: addScriptFile(path, new File(path)), ensuring that a given path is
+ * added to the jar at most once.
+ * @param path
+ * @throws MalformedURLException
+ */
public void addScriptFile(String path) throws MalformedURLException {
if (path != null) {
- scriptFiles.add(path);
+ aliasedScriptFiles.put(path.replaceFirst("^/", ""), new File(path));
}
}
-
+
+ /**
+ * this method adds script files that must be added to the shipped jar
+ * named differently from their local fs path.
+ * @param name name in the jar
+ * @param path path on the local fs
+ */
+ public void addScriptFile(String name, String path) {
+ if (path != null) {
+ aliasedScriptFiles.put(name.replaceFirst("^/", ""), new File(path));
+ }
+ }
+
public void addJar(String path) throws MalformedURLException {
if (path != null) {
URL resource = (new File(path)).toURI().toURL();
@@ -234,6 +255,14 @@ public class PigContext implements Seria
}
}
+ /**
+ * script files as name/file pairs to be added to the job jar
+ * @return name/file pairs
+ */
+ public Map<String,File> getScriptFiles() {
+ return aliasedScriptFiles;
+ }
+
public void rename(String oldName, String newName) throws IOException {
if (oldName.equals(newName)) {
return;
Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/util/JarManager.java Tue Aug 7 17:47:08 2012
@@ -128,6 +128,18 @@ public class JarManager {
String path = pigContext.scriptFiles.get(i);
addStream(jarFile, path, new FileInputStream(new File(path)),contents);
}
+ for (Map.Entry<String, File> entry : pigContext.getScriptFiles().entrySet()) {
+ InputStream stream = null;
+ if (entry.getValue().exists()) {
+ stream = new FileInputStream(entry.getValue());
+ } else {
+ stream = PigContext.getClassLoader().getResourceAsStream(entry.getValue().getPath());
+ }
+ if (stream==null) {
+ throw new IOException("Cannot find " + entry.getValue().getPath());
+ }
+ addStream(jarFile, entry.getKey(), stream, contents);
+ }
jarFile.putNextEntry(new ZipEntry("pigContext"));
new ObjectOutputStream(jarFile).writeObject(pigContext);
Modified: pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java?rev=1370393&r1=1370392&r2=1370393&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/scripting/ScriptEngine.java Tue Aug 7 17:47:08 2012
@@ -132,10 +132,23 @@ public abstract class ScriptEngine {
throw new IllegalStateException("could not find existing file "+scriptPath, e);
}
} else {
- if (file.isAbsolute()) {
- is = ScriptEngine.class.getResourceAsStream(scriptPath);
- } else {
- is = ScriptEngine.class.getResourceAsStream("/" + scriptPath);
+ // Try system, current and context classloader.
+ is = ScriptEngine.class.getResourceAsStream(scriptPath);
+ if (is == null) {
+ is = getResourceUsingClassLoader(scriptPath, ScriptEngine.class.getClassLoader());
+ }
+ if (is == null) {
+ is = getResourceUsingClassLoader(scriptPath, Thread.currentThread().getContextClassLoader());
+ }
+ if (is == null && !file.isAbsolute()) {
+ String path = "/" + scriptPath;
+ is = ScriptEngine.class.getResourceAsStream(path);
+ if (is == null) {
+ is = getResourceUsingClassLoader(path, ScriptEngine.class.getClassLoader());
+ }
+ if (is == null) {
+ is = getResourceUsingClassLoader(path, Thread.currentThread().getContextClassLoader());
+ }
}
}
@@ -148,6 +161,13 @@ public abstract class ScriptEngine {
return is;
}
+ private static InputStream getResourceUsingClassLoader(String fullFilename, ClassLoader loader) {
+ if (loader != null) {
+ return loader.getResourceAsStream(fullFilename);
+ }
+ return null;
+ }
+
public static final String NAMESPACE_SEPARATOR = ".";
/**