You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/01/24 19:42:20 UTC
svn commit: r1062919 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/builtin/Utf8StorageConverter.java
test/org/apache/pig/test/TestConversions.java
Author: daijy
Date: Mon Jan 24 18:42:19 2011
New Revision: 1062919
URL: http://svn.apache.org/viewvc?rev=1062919&view=rev
Log:
PIG-496: project of bags from complex data causes failures
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
pig/trunk/test/org/apache/pig/test/TestConversions.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1062919&r1=1062918&r2=1062919&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Jan 24 18:42:19 2011
@@ -86,6 +86,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-496: project of bags from complex data causes failures (daijy)
+
PIG-730: problem combining schema from a union of several LOAD expressions, with a nested bag inside the schema (daijy)
PIG-767: Schema reported from DESCRIBE and actual schema of inner bags are different (daijy)
Modified: pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java?rev=1062919&r1=1062918&r2=1062919&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java Mon Jan 24 18:42:19 2011
@@ -73,7 +73,7 @@ public class Utf8StorageConverter implem
if (fieldSchema==null) {
throw new IOException("Schema is null");
}
- ResourceFieldSchema[] fss=fieldSchema.getSchema().getFields();;
+ ResourceFieldSchema[] fss=fieldSchema.getSchema().getFields();
Tuple t;
int buf;
while ((buf=in.read())!='{') {
@@ -161,20 +161,31 @@ public class Utf8StorageConverter implem
}
if (buf=='['||buf=='{'||buf=='(') {
level.push((char)buf);
+ mOut.write(buf);
}
else if (buf==')' && level.isEmpty()) // End of tuple
+ {
+ DataByteArray value = new DataByteArray(mOut.toByteArray());
+ t.append(value);
break;
+ }
+ else if (buf==',' && level.isEmpty())
+ {
+ DataByteArray value = new DataByteArray(mOut.toByteArray());
+ t.append(value);
+ mOut.reset();
+ }
else if (buf==']' ||buf=='}'||buf==')')
{
if (level.peek()==findStartChar((char)buf))
level.pop();
else
throw new IOException("Malformed tuple");
+ mOut.write(buf);
}
- mOut.write(buf);
+ else
+ mOut.write(buf);
}
- DataByteArray value = new DataByteArray(mOut.toByteArray());
- t.append(value);
}
return t;
}
Modified: pig/trunk/test/org/apache/pig/test/TestConversions.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestConversions.java?rev=1062919&r1=1062918&r2=1062919&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestConversions.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestConversions.java Mon Jan 24 18:42:19 2011
@@ -345,9 +345,11 @@ public class TestConversions extends Tes
schema = Utils.getSchemaFromString("t:tuple()");
rfs = new ResourceSchema(schema).getFields()[0];
t = ps.getLoadCaster().bytesToTuple(s.getBytes(), rfs);
- assertTrue(t.size()==1);
+ assertTrue(t.size()==2);
assertTrue(t.get(0) instanceof DataByteArray);
- assertTrue(t.get(0).toString().equals("a,b"));
+ assertTrue(t.get(0).toString().equals("a"));
+ assertTrue(t.get(1) instanceof DataByteArray);
+ assertTrue(t.get(1).toString().equals("b"));
s = "[a#(1,2,3)]";
m = ps.getLoadCaster().bytesToMap(s.getBytes());
@@ -360,9 +362,13 @@ public class TestConversions extends Tes
schema = Utils.getSchemaFromString("t:tuple()");
rfs = new ResourceSchema(schema).getFields()[0];
t = ps.getLoadCaster().bytesToTuple(s.getBytes(), rfs);
- assertTrue(t.size()==1);
+ assertTrue(t.size()==3);
assertTrue(t.get(0) instanceof DataByteArray);
- assertTrue(t.get(0).toString().equals("a,b,(123,456,{(1,2,3)})"));
+ assertTrue(t.get(0).toString().equals("a"));
+ assertTrue(t.get(1) instanceof DataByteArray);
+ assertTrue(t.get(1).toString().equals("b"));
+ assertTrue(t.get(2) instanceof DataByteArray);
+ assertTrue(t.get(2).toString().equals("(123,456,{(1,2,3)})"));
s = "(a,b,(123,456,{(1,2,3}))";
schema = Utils.getSchemaFromString("t:tuple()");