You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/03/17 02:17:44 UTC
svn commit: r1301849 - in /pig/branches/branch-0.10: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/
contrib/piggybank/java/src/test/java/or...
Author: daijy
Date: Sat Mar 17 01:17:44 2012
New Revision: 1301849
URL: http://svn.apache.org/viewvc?rev=1301849&view=rev
Log:
PIG-2411: AvroStorage UDF in PiggyBank fails to STORE a bag of single-field tuples as Avro arrays
Added:
pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/messages.avro (with props)
Modified:
pig/branches/branch-0.10/CHANGES.txt
pig/branches/branch-0.10/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Modified: pig/branches/branch-0.10/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/CHANGES.txt?rev=1301849&r1=1301848&r2=1301849&view=diff
==============================================================================
--- pig/branches/branch-0.10/CHANGES.txt (original)
+++ pig/branches/branch-0.10/CHANGES.txt Sat Mar 17 01:17:44 2012
@@ -176,6 +176,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2411: AvroStorage UDF in PiggyBank fails to STORE a bag of single-field tuples as Avro arrays (russell.jurney via daijy)
+
PIG-2588: e2e harness: use pig command for cluster deploy (thw via daijy)
PIG-2565: Support IMPORT for macros stored in S3 Buckets (daijy)
Modified: pig/branches/branch-0.10/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java?rev=1301849&r1=1301848&r2=1301849&view=diff
==============================================================================
--- pig/branches/branch-0.10/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java (original)
+++ pig/branches/branch-0.10/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java Sat Mar 17 01:17:44 2012
@@ -324,8 +324,12 @@ public class AvroStorageUtils {
/** check whether it is just a wrapped tuple */
public static boolean isTupleWrapper(ResourceFieldSchema pigSchema) {
- return pigSchema.getType() == DataType.TUPLE
- && pigSchema.getName().equals(AvroStorageUtils.PIG_TUPLE_WRAPPER);
+ Boolean status = false;
+ if(pigSchema.getType() == DataType.TUPLE)
+ if(pigSchema.getName() != null)
+ if(pigSchema.getName().equals(AvroStorageUtils.PIG_TUPLE_WRAPPER))
+ status = true;
+ return status;
}
/** extract schema from a nullable union */
Modified: pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1301849&r1=1301848&r2=1301849&view=diff
==============================================================================
--- pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Sat Mar 17 01:17:44 2012
@@ -68,6 +68,7 @@ public class TestAvroStorage {
final private String testRecordFile = getInputFile("test_record.avro");
final private String testRecordSchema = getInputFile("test_record.avsc");
final private String testTextFile = getInputFile("test_record.txt");
+ final private String testSingleTupleBagFile = getInputFile("messages.avro");
@BeforeClass
public static void setup() throws ExecException {
@@ -292,6 +293,20 @@ public class TestAvroStorage {
testAvroStorage( queries);
verifyResults(output, expected);
}
+
+ @Test
+ public void testSingleFieldTuples() throws IOException {
+ String output= outbasedir + "testSingleFieldTuples";
+ String expected = basedir + "expected_testSingleFieldTuples.avro";
+ deleteDirectory(new File(output));
+ String [] queries = {
+ " messages = LOAD '" + testSingleTupleBagFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " a = foreach (group messages by user_id) { sorted = order messages by message_id DESC; GENERATE group AS user_id, sorted AS messages; };",
+ " STORE a INTO '" + output + "' " +
+ " USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
+ };
+ testAvroStorage( queries);
+ }
private static void deleteDirectory (File path) {
if ( path.exists()) {
Added: pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/messages.avro
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/messages.avro?rev=1301849&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/branches/branch-0.10/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/messages.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream