You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by sm...@apache.org on 2012/08/06 09:44:07 UTC
svn commit: r1369740 - in /pig/trunk: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/pi...
Author: sms
Date: Mon Aug 6 07:44:06 2012
New Revision: 1369740
URL: http://svn.apache.org/viewvc?rev=1369740&view=rev
Log:
PIG-2837: AvroStorage throws StackOverFlowError (cheolsoo via sms)
Added:
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_generic_union_schema.avro (with props)
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_recursive_schema.avro (with props)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1369740&r1=1369739&r2=1369740&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Aug 6 07:44:06 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2837: AvroStorage throws StackOverFlowError (cheolsoo via sms)
+
PIG-2856: AvroStorage doesn't load files in the directories when a glob pattern matches both files and directories. (cheolsoo via sms)
PIG-2569: Fix org.apache.pig.test.TestInvoker.testSpeed (aklochkov via dvryaboy)
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java?rev=1369740&r1=1369739&r2=1369740&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroSchema2Pig.java Mon Aug 6 07:44:06 2012
@@ -47,12 +47,12 @@ public class AvroSchema2Pig {
*/
public static ResourceSchema convert(Schema schema) throws IOException {
- if (AvroStorageUtils.containsGenericUnion(schema))
- throw new IOException ("We don't accept schema containing generic unions.");
-
if (AvroStorageUtils.containsRecursiveRecord(schema))
throw new IOException ("We don't accept schema containing recursive records.");
+ if (AvroStorageUtils.containsGenericUnion(schema))
+ throw new IOException ("We don't accept schema containing generic unions.");
+
ResourceFieldSchema inSchema = inconvert(schema, FIELD);
ResourceSchema tupleSchema;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1369740&r1=1369739&r2=1369740&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Mon Aug 6 07:44:06 2012
@@ -32,6 +32,7 @@ import org.apache.pig.backend.executione
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobCreationException;
+import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.piggybank.storage.avro.AvroStorage;
import org.apache.pig.piggybank.storage.avro.PigSchema2Avro;
import org.apache.pig.test.Util;
@@ -81,6 +82,8 @@ public class TestAvroStorage {
final private String testArrayFile = getInputFile("test_array.avro");
final private String testRecordFile = getInputFile("test_record.avro");
final private String testRecordSchema = getInputFile("test_record.avsc");
+ final private String testRecursiveSchemaFile = getInputFile("test_recursive_schema.avro");
+ final private String testGenericUnionSchemaFile = getInputFile("test_generic_union_schema.avro");
final private String testTextFile = getInputFile("test_record.txt");
final private String testSingleTupleBagFile = getInputFile("messages.avro");
final private String testNoExtensionFile = getInputFile("test_no_extension");
@@ -97,6 +100,48 @@ public class TestAvroStorage {
}
@Test
+ public void testRecursiveSchema() throws IOException {
+ // Verify that a FrontendException is thrown if schema is recursive.
+ String output= outbasedir + "testRecursiveSchema";
+ deleteDirectory(new File(output));
+ String [] queries = {
+ " in = LOAD '" + testRecursiveSchemaFile +
+ "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " STORE in INTO '" + output +
+ "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
+ };
+ try {
+ testAvroStorage(queries);
+ Assert.fail();
+ } catch (FrontendException e) {
+ // The IOException thrown by AvroStorage for recursive schema is caught
+ // by the Pig frontend, and FrontendException is re-thrown.
+ assertTrue(e.getMessage().contains("Cannot get schema"));
+ }
+ }
+
+ @Test
+ public void testGenericUnionSchema() throws IOException {
+ // Verify that a FrontendException is thrown if schema has generic union.
+ String output= outbasedir + "testGenericUnionSchema";
+ deleteDirectory(new File(output));
+ String [] queries = {
+ " in = LOAD '" + testGenericUnionSchemaFile +
+ "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " STORE in INTO '" + output +
+ "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
+ };
+ try {
+ testAvroStorage(queries);
+ Assert.fail();
+ } catch (FrontendException e) {
+ // The IOException thrown by AvroStorage for generic union is caught
+ // by the Pig frontend, and FrontendException is re-thrown.
+ assertTrue(e.getMessage().contains("Cannot get schema"));
+ }
+ }
+
+ @Test
public void testDir() throws IOException {
// Verify that all files in a directory including its sub-directories are loaded.
String output= outbasedir + "testDir";
Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_generic_union_schema.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_generic_union_schema.avro?rev=1369740&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_generic_union_schema.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_recursive_schema.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_recursive_schema.avro?rev=1369740&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_recursive_schema.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream