You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ju...@apache.org on 2011/04/06 23:39:31 UTC
svn commit: r1089641 - in /pig/trunk: ./ src/org/apache/pig/parser/
src/org/apache/pig/scripting/js/ src/org/apache/pig/scripting/jython/
test/org/apache/pig/test/
Author: julien
Date: Wed Apr 6 21:39:31 2011
New Revision: 1089641
URL: http://svn.apache.org/viewvc?rev=1089641&view=rev
Log:
PIG-1923 Map of primitive types in Python and PIG-1944 javascript register udfs
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/PlanGenerationFailureException.java
pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java
pig/trunk/src/org/apache/pig/scripting/jython/JythonFunction.java
pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java
pig/trunk/test/org/apache/pig/test/TestScriptUDF.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1089641&r1=1089640&r2=1089641&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Apr 6 21:39:31 2011
@@ -136,6 +136,10 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1923: Jython UDFs fail to convert Maps of Integer values back to Pig types (julien)
+
+PIG-1944: register javascript UDFs does not work (julien)
+
PIG-1955: PhysicalOperator has a member variable (non-static) Log object that
is non-transient, this causes serialization errors (woody via rding)
Modified: pig/trunk/src/org/apache/pig/parser/PlanGenerationFailureException.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/PlanGenerationFailureException.java?rev=1089641&r1=1089640&r2=1089641&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/PlanGenerationFailureException.java (original)
+++ pig/trunk/src/org/apache/pig/parser/PlanGenerationFailureException.java Wed Apr 6 21:39:31 2011
@@ -38,4 +38,8 @@ public class PlanGenerationFailureExcept
return ex;
}
+ @Override
+ public Throwable getCause() {
+ return ex;
+ }
}
Modified: pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java?rev=1089641&r1=1089640&r2=1089641&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/js/JsScriptEngine.java Wed Apr 6 21:39:31 2011
@@ -217,10 +217,6 @@ public class JsScriptEngine extends Scri
throw new IOException("Can't read file: " + scriptFile);
}
- // to enable passing of information to the slave
- this.scriptPath = scriptFile;
- clientInstance = this;
-
registerFunctions(scriptFile, null, pigContext);
// run main
@@ -232,6 +228,10 @@ public class JsScriptEngine extends Scri
@Override
public void registerFunctions(String path, String namespace,
PigContext pigContext) throws IOException {
+ // to enable passing of information to the slave
+ this.scriptPath = path;
+ this.clientInstance = this;
+
pigContext.scriptJars.add(getJarPath(Context.class));
namespace = (namespace == null) ? "" : namespace + NAMESPACE_SEPARATOR;
FileInputStream fis = new FileInputStream(path);
Modified: pig/trunk/src/org/apache/pig/scripting/jython/JythonFunction.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/jython/JythonFunction.java?rev=1089641&r1=1089640&r2=1089641&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/jython/JythonFunction.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/jython/JythonFunction.java Wed Apr 6 21:39:31 2011
@@ -80,7 +80,7 @@ public class JythonFunction extends Eval
"unregistered " + functionName);
}
} catch (ParseException pe) {
- throw new ExecException("Could not parse schema for script function " + pe);
+ throw new ExecException("Could not parse schema for script function " + pe, pe);
} catch (IOException e) {
throw new IllegalStateException("Could not initialize: " + filename, e);
} catch (Exception e) {
Modified: pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java?rev=1089641&r1=1089640&r2=1089641&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java Wed Apr 6 21:39:31 2011
@@ -69,10 +69,15 @@ public class JythonUtils {
}
javaObj = list;
} else if (pyObject instanceof PyDictionary) {
- Map<?, PyObject> map = Py.tojava(pyObject, Map.class);
+ Map<?, Object> map = Py.tojava(pyObject, Map.class);
Map<Object, Object> newMap = new HashMap<Object, Object>();
- for (Map.Entry<?, PyObject> entry : map.entrySet()) {
- newMap.put(entry.getKey(), pythonToPig(entry.getValue()));
+ for (Map.Entry<?, Object> entry : map.entrySet()) {
+ if (entry.getValue() instanceof PyObject) {
+ newMap.put(entry.getKey(), pythonToPig((PyObject) entry.getValue()));
+ } else {
+ // Jython sometimes uses directly the java class: for example for integers
+ newMap.put(entry.getKey(), entry.getValue());
+ }
}
javaObj = newMap;
} else if (pyObject instanceof PyLong) {
@@ -94,15 +99,15 @@ public class JythonUtils {
javaObj = new DataByteArray((byte[])javaObj);
}
else {
- throw new ExecException("Non supported pig datatype found, cast failed");
+ throw new ExecException("Non supported pig datatype found, cast failed: "+(pyObject==null?null:pyObject.getClass().getName()));
}
}
if(javaObj.equals(Py.NoConversion)) {
- throw new ExecException("Cannot cast into any pig supported type");
+ throw new ExecException("Cannot cast into any pig supported type: "+(pyObject==null?null:pyObject.getClass().getName()));
}
return javaObj;
} catch (Exception e) {
- throw new ExecException("Cannot convert jython type to pig datatype "+ e);
+ throw new ExecException("Cannot convert jython type ("+(pyObject==null?null:pyObject.getClass().getName())+") to pig datatype "+ e, e);
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestScriptUDF.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestScriptUDF.java?rev=1089641&r1=1089640&r2=1089641&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestScriptUDF.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestScriptUDF.java Wed Apr 6 21:39:31 2011
@@ -19,6 +19,7 @@ package org.apache.pig.test;
import java.io.File;
import java.util.Iterator;
+import java.util.Map;
import java.util.Random;
import junit.framework.TestCase;
@@ -112,6 +113,50 @@ public class TestScriptUDF extends TestC
assertTrue(t.toString().equals("(9)"));
}
+
+ @Test
+ public void testJavascriptExampleScript() throws Exception{
+ String[] script = {
+ "helloworld.outputSchema = \"word:chararray\";",
+ "function helloworld() {",
+ "return 'Hello, World';",
+ "}",
+ "complex.outputSchema = \"word:chararray,num:long\";",
+ "function complex(word) {",
+ "return {word:word, num:word.length};",
+ "}",
+ };
+ String[] input = {
+ "one\t1",
+ "two\t2",
+ "three\t3"
+ };
+
+ Util.createInputFile(cluster, "table_testJavascriptExampleScript", input);
+ Util.createLocalInputFile( "testJavascriptExampleScript.js", script);
+
+ // Test the namespace
+ pigServer.registerCode("testJavascriptExampleScript.js", "javascript", "myfuncs");
+ pigServer.registerQuery("A = LOAD 'table_testJavascriptExampleScript' as (a0:chararray, a1:long);");
+ pigServer.registerQuery("B = foreach A generate myfuncs.helloworld(), myfuncs.complex($0);");
+
+ Iterator<Tuple> iter = pigServer.openIterator("B");
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+
+ assertEquals(((Tuple)t.get(1)).get(1), 3);
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+
+ assertEquals(((Tuple)t.get(1)).get(1), 3);
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+
+ assertEquals(((Tuple)t.get(1)).get(1), 5);
+
+ }
// See PIG-928
@Test
@@ -347,6 +392,45 @@ public class TestScriptUDF extends TestC
}
@Test
+ public void testPythonScriptUDFMapOutput() throws Exception{
+ String[] script = {
+ "#!/usr/bin/python",
+ "@outputSchema(\"mapint:[]\")",
+ "def maptomapint(map):" ,
+ "\toutMap = {}",
+ "\tfor k, v in map.iteritems():",
+ "\t\toutMap[k] = len(v)",
+ "\treturn outMap"
+ };
+ String[] input = {
+ "[1#hello,2#world]",
+ "[3#pig,4#rocks]",
+ };
+
+ Util.createInputFile(cluster, "table_testPythonScriptUDFMapOutput", input);
+ Util.createLocalInputFile( "testPythonScriptUDFMapOutput.py", script);
+
+ pigServer.registerCode("testPythonScriptUDFMapOutput.py", "jython", "pig");
+ pigServer.registerQuery("A = LOAD 'table_testPythonScriptUDFMapOutput' as (a0:map[]);");
+ pigServer.registerQuery("B = foreach A generate pig.maptomapint(a0);");
+
+ Iterator<Tuple> iter = pigServer.openIterator("B");
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+
+ assertEquals(5, ((Map<?,?>)t.get(0)).get("1"));
+ assertEquals(5, ((Map<?,?>)t.get(0)).get("2"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertEquals(3, ((Map<?,?>)t.get(0)).get("3"));
+ assertEquals(5, ((Map<?,?>)t.get(0)).get("4"));
+
+ assertFalse(iter.hasNext());
+
+ }
+
+ @Test
public void testPythonScriptUDFNullInputOutput() throws Exception {
String[] script = {
"#!/usr/bin/python",