You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/06/26 22:22:44 UTC
svn commit: r1354205 - in /pig/branches/branch-0.10: ./ src/org/apache/pig/
src/org/apache/pig/impl/ src/org/apache/pig/scripting/
test/e2e/pig/udfs/python/
Author: daijy
Date: Tue Jun 26 20:22:43 2012
New Revision: 1354205
URL: http://svn.apache.org/viewvc?rev=1354205&view=rev
Log:
PIG-2761: With hadoop23 importing modules inside python script does not work
Added:
pig/branches/branch-0.10/test/e2e/pig/udfs/python/stringutil.py
Modified:
pig/branches/branch-0.10/CHANGES.txt
pig/branches/branch-0.10/src/org/apache/pig/PigServer.java
pig/branches/branch-0.10/src/org/apache/pig/impl/PigContext.java
pig/branches/branch-0.10/src/org/apache/pig/scripting/ScriptEngine.java
pig/branches/branch-0.10/test/e2e/pig/udfs/python/scriptingudf.py
Modified: pig/branches/branch-0.10/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/CHANGES.txt?rev=1354205&r1=1354204&r2=1354205&view=diff
==============================================================================
--- pig/branches/branch-0.10/CHANGES.txt (original)
+++ pig/branches/branch-0.10/CHANGES.txt Tue Jun 26 20:22:43 2012
@@ -28,6 +28,8 @@ PIG-2727: PigStorage Source tagging does
BUG FIXES
+PIG-2761: With hadoop23 importing modules inside python script does not work (rohini via daijy)
+
PIG-2759: Typo in document "Built In Functions" (daijy)
PIG-2745: Pig e2e test RubyUDFs fails in MR mode when running from tarball (cheolsoo via daijy)
Modified: pig/branches/branch-0.10/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/PigServer.java?rev=1354205&r1=1354204&r2=1354205&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.10/src/org/apache/pig/PigServer.java Tue Jun 26 20:22:43 2012
@@ -519,13 +519,16 @@ public class PigServer {
throw new FrontendException(msg, errCode,
PigException.USER_ENVIRONMENT);
}
+ String cwd = new File(".").getCanonicalPath();
+ String filePath = f.getCanonicalPath();
+ //Use the relative path in the jar, if the path specified is relative
+ String nameInJar = filePath.equals(cwd + File.separator + path) ?
+ filePath.substring(cwd.length() + 1) : filePath;
+ pigContext.addScriptFile(nameInJar, filePath);
if(scriptingLang != null) {
ScriptEngine se = ScriptEngine.getInstance(scriptingLang);
- se.registerFunctions(f.getPath(), namespace, pigContext);
+ se.registerFunctions(nameInJar, namespace, pigContext);
}
- // Remove the leading "/" or "./" from the name of the file to store in jar.
- String nameInJar = f.getPath().replaceFirst("^\\./|^/", "");
- pigContext.addScriptFile(nameInJar, f.getPath());
}
/**
Modified: pig/branches/branch-0.10/src/org/apache/pig/impl/PigContext.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/impl/PigContext.java?rev=1354205&r1=1354204&r2=1354205&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/branches/branch-0.10/src/org/apache/pig/impl/PigContext.java Tue Jun 26 20:22:43 2012
@@ -244,7 +244,7 @@ public class PigContext implements Seria
*/
public void addScriptFile(String path) throws MalformedURLException {
if (path != null) {
- aliasedScriptFiles.put(path, new File(path));
+ aliasedScriptFiles.put(path.replaceFirst("^/", ""), new File(path));
}
}
@@ -256,7 +256,7 @@ public class PigContext implements Seria
*/
public void addScriptFile(String name, String path) {
if (path != null) {
- aliasedScriptFiles.put(name, new File(path));
+ aliasedScriptFiles.put(name.replaceFirst("^/", ""), new File(path));
}
}
Modified: pig/branches/branch-0.10/src/org/apache/pig/scripting/ScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/src/org/apache/pig/scripting/ScriptEngine.java?rev=1354205&r1=1354204&r2=1354205&view=diff
==============================================================================
--- pig/branches/branch-0.10/src/org/apache/pig/scripting/ScriptEngine.java (original)
+++ pig/branches/branch-0.10/src/org/apache/pig/scripting/ScriptEngine.java Tue Jun 26 20:22:43 2012
@@ -132,10 +132,23 @@ public abstract class ScriptEngine {
throw new IllegalStateException("could not find existing file "+scriptPath, e);
}
} else {
- if (file.isAbsolute()) {
- is = ScriptEngine.class.getResourceAsStream(scriptPath);
- } else {
- is = ScriptEngine.class.getResourceAsStream("/" + scriptPath);
+ // Try system, current and context classloader.
+ is = ScriptEngine.class.getResourceAsStream(scriptPath);
+ if (is == null) {
+ is = getResourceUsingClassLoader(scriptPath, ScriptEngine.class.getClassLoader());
+ }
+ if (is == null) {
+ is = getResourceUsingClassLoader(scriptPath, Thread.currentThread().getContextClassLoader());
+ }
+ if (is == null && !file.isAbsolute()) {
+ String path = "/" + scriptPath;
+ is = ScriptEngine.class.getResourceAsStream(path);
+ if (is == null) {
+ is = getResourceUsingClassLoader(path, ScriptEngine.class.getClassLoader());
+ }
+ if (is == null) {
+ is = getResourceUsingClassLoader(path, Thread.currentThread().getContextClassLoader());
+ }
}
}
@@ -148,6 +161,13 @@ public abstract class ScriptEngine {
return is;
}
+ private static InputStream getResourceUsingClassLoader(String fullFilename, ClassLoader loader) {
+ if (loader != null) {
+ return loader.getResourceAsStream(fullFilename);
+ }
+ return null;
+ }
+
public static final String NAMESPACE_SEPARATOR = ".";
/**
Modified: pig/branches/branch-0.10/test/e2e/pig/udfs/python/scriptingudf.py
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/e2e/pig/udfs/python/scriptingudf.py?rev=1354205&r1=1354204&r2=1354205&view=diff
==============================================================================
--- pig/branches/branch-0.10/test/e2e/pig/udfs/python/scriptingudf.py (original)
+++ pig/branches/branch-0.10/test/e2e/pig/udfs/python/scriptingudf.py Tue Jun 26 20:22:43 2012
@@ -15,7 +15,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+
+import sys
+sys.path.append("./libexec/python")
+import stringutil
+
@outputSchemaFunction("squareSchema")
def square(num):
return ((num)*(num))
@@ -48,4 +52,4 @@ def isretired(age):
outputSchema("words:{(word:chararray)}")
def tokenize(sentence):
- return sentence.split(' ')
+ return stringutil.tokenize(sentence)
Added: pig/branches/branch-0.10/test/e2e/pig/udfs/python/stringutil.py
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/test/e2e/pig/udfs/python/stringutil.py?rev=1354205&view=auto
==============================================================================
--- pig/branches/branch-0.10/test/e2e/pig/udfs/python/stringutil.py (added)
+++ pig/branches/branch-0.10/test/e2e/pig/udfs/python/stringutil.py Tue Jun 26 20:22:43 2012
@@ -0,0 +1,20 @@
+#!/usr/bin/python
+
+############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def tokenize(sentence):
+ return sentence.split(' ')