You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/06/06 07:08:47 UTC
svn commit: r1346734 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/scripting/jython/JythonUtils.java
test/e2e/pig/tests/nightly.conf test/e2e/pig/udfs/python/scriptingudf.py
Author: daijy
Date: Wed Jun 6 05:08:46 2012
New Revision: 1346734
URL: http://svn.apache.org/viewvc?rev=1346734&view=rev
Log:
PIG-2739: PyList should map to Bag automatically in Jython
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java
pig/trunk/test/e2e/pig/tests/nightly.conf
pig/trunk/test/e2e/pig/udfs/python/scriptingudf.py
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1346734&r1=1346733&r2=1346734&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 6 05:08:46 2012
@@ -385,6 +385,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2739: PyList should map to Bag automatically in Jython (daijy)
+
PIG-2730: TFileStorage getStatistics incorrectly throws an exception instead of returning null (traviscrawford via daijy)
PIG-2717: Tuple field mangled during flattening (daijy)
Modified: pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java?rev=1346734&r1=1346733&r2=1346734&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/jython/JythonUtils.java Wed Jun 6 05:08:46 2012
@@ -63,10 +63,17 @@ public class JythonUtils {
} else if (pyObject instanceof PyList) {
DataBag list = bagFactory.newDefaultBag();
for (PyObject bagTuple : ((PyList) pyObject).asIterable()) {
- // In jython, list need not be a bag of tuples, as it is in case of pig
- // So we fail with cast exception if we dont find tuples inside bag
- // This is consistent with java udf (bag should be filled with tuples)
- list.add((Tuple) pythonToPig(bagTuple));
+ // If the item of the array is not a tuple,
+ // wrap it into tuple before adding to bag
+ Object pigBagItem = pythonToPig(bagTuple);
+ Tuple pigBagTuple;
+ if (!(pigBagItem instanceof Tuple)) {
+ pigBagTuple = TupleFactory.getInstance().newTuple(1);
+ pigBagTuple.set(0, pigBagItem);
+ } else {
+ pigBagTuple = (Tuple)pigBagItem;
+ }
+ list.add(pigBagTuple);
}
javaObj = list;
} else if (pyObject instanceof PyDictionary) {
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1346734&r1=1346733&r2=1346734&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Wed Jun 6 05:08:46 2012
@@ -3459,6 +3459,23 @@ store b into ':OUTPATH:';\,
a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:int, gpa:double);
b = foreach a generate name, (age>=60?'true':'false');
store b into ':OUTPATH:';\,
+ },
+ {
+ # jython udf which returns an array
+ 'num' => 12,
+ 'pig' => q\
+register ':SCRIPTHOMEPATH:/python/scriptingudf.py' using jython as myfuncs;
+a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age:chararray, gpa:chararray);
+b = foreach a generate CONCAT(CONCAT(age, ' '), gpa) as sentence;
+c = foreach b generate flatten(myfuncs.tokenize(sentence));
+store c into ':OUTPATH:';\,
+ 'verify_pig_script' => q\
+a = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:chararray, gpa:chararray);
+b = load ':INPATH:/singlefile/studenttab10k' using PigStorage() as (name:chararray, age:chararray, gpa:chararray);
+c = foreach a generate age;
+d = foreach b generate gpa;
+e = union c, d;
+store e into ':OUTPATH:';\,
}
]
},
Modified: pig/trunk/test/e2e/pig/udfs/python/scriptingudf.py
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/udfs/python/scriptingudf.py?rev=1346734&r1=1346733&r2=1346734&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/udfs/python/scriptingudf.py (original)
+++ pig/trunk/test/e2e/pig/udfs/python/scriptingudf.py Wed Jun 6 05:08:46 2012
@@ -87,3 +87,7 @@ def isretired(age):
return True
else:
return False
+
+outputSchema("words:{(word:chararray)}")
+def tokenize(sentence):
+ return sentence.split(' ')