You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/06/04 03:03:26 UTC
svn commit: r1489264 - in /pig/trunk: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/pi...
Author: rohini
Date: Tue Jun 4 01:03:26 2013
New Revision: 1489264
URL: http://svn.apache.org/r1489264
Log:
PIG-3322: AvroStorage give NPE on reading file with union as top level schema (viraj via rohini)
Added:
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_loadavrowithnulls.avro (with props)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroRecordReader.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1489264&r1=1489263&r2=1489264&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jun 4 01:03:26 2013
@@ -192,6 +192,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3322: AvroStorage give NPE on reading file with union as top level schema (viraj via rohini)
+
PIG-2828: Handle nulls in DataType.compare (aniket486)
PIG-3335: TestErrorHandling.tesNegative7 fails on MR2 (xuefuz)
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java?rev=1489264&r1=1489263&r2=1489264&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java Tue Jun 4 01:03:26 2013
@@ -534,7 +534,6 @@ public class AvroStorage extends FileInp
AvroStorageLog.details("data path=" + path.toUri().toString());
FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
outputAvroSchema = getAvroSchema(path, fs);
- userSpecifiedAvroSchema = outputAvroSchema;
} else if (name.equalsIgnoreCase("nullable")) {
nullable = (Boolean) value;
} else if (name.equalsIgnoreCase("schema")) {
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroRecordReader.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroRecordReader.java?rev=1489264&r1=1489263&r2=1489264&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroRecordReader.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroRecordReader.java Tue Jun 4 01:03:26 2013
@@ -141,7 +141,12 @@ public class PigAvroRecordReader extends
AvroStorageLog.details("Class =" + obj.getClass());
result = (Tuple) obj;
} else {
- AvroStorageLog.details("Wrap calss " + obj.getClass() + " as a tuple.");
+ if (obj != null) {
+ AvroStorageLog.details("Wrap class " + obj.getClass() + " as a tuple.");
+ }
+ else {
+ AvroStorageLog.details("Wrap null as a tuple.");
+ }
result = wrapAsTuple(obj);
}
if (schemaToMergedSchemaMap != null) {
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1489264&r1=1489263&r2=1489264&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Tue Jun 4 01:03:26 2013
@@ -16,6 +16,10 @@
*/
package org.apache.pig.piggybank.test.storage.avro;
+import static org.apache.pig.builtin.mock.Storage.resetData;
+import static org.apache.pig.builtin.mock.Storage.schema;
+import static org.apache.pig.builtin.mock.Storage.tuple;
+
import org.apache.avro.file.DataFileStream;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.commons.logging.Log;
@@ -33,6 +37,8 @@ import org.apache.pig.backend.executione
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobCreationException;
+import org.apache.pig.builtin.mock.Storage.Data;
+import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.piggybank.storage.avro.AvroStorage;
@@ -181,6 +187,7 @@ public class TestAvroStorage {
final private String testMultipleSchemas1File = getInputFile("test_primitive_types/*");
final private String testMultipleSchemas2File = getInputFile("test_complex_types/*");
final private String testUserDefinedLoadSchemaFile = getInputFile("test_user_defined_load_schema/*");
+ final private String testLoadwithNullValues = getInputFile("test_loadavrowithnulls.avro");
@BeforeClass
public static void setup() throws ExecException, IOException {
@@ -1075,6 +1082,39 @@ public class TestAvroStorage {
verifyResults(output, expected);
}
+ @Test
+ // Schema for the generated avro file test_loadavrowithnulls.avro
+ // ["null",{"type":"record","name":"TUPLE_0",
+ // "fields":[
+ // {"name":"name","type":["null","string"],"doc":"autogenerated from Pig Field Schema"},
+ // {"name":"age","type":["null","int"],"doc":"autogenerated from Pig Field Schema"},
+ // {"name":"gpa","type":["null","double"],"doc":"autogenerated from Pig Field Schema"}]}]
+ public void testLoadwithNullValues() throws IOException {
+ //Input is supposed to have empty tuples
+ PigSchema2Avro.setTupleIndex(0);
+ Data data = resetData(pigServerLocal);
+ String output = outbasedir + "testLoadwithNulls";
+ deleteDirectory(new File(output));
+ String [] queries = {
+ " A = load '" + testLoadwithNullValues + "' USING " +
+ " org.apache.pig.piggybank.storage.avro.AvroStorage(); ",
+ " B = order A by name;",
+ " store B into '" + output +"' USING mock.Storage();"
+ };
+ testAvroStorage(queries);
+ List<Tuple> out = data.get(output);
+ assertEquals(out + " size", 4, out.size());
+
+ assertEquals(schema("name:chararray,age:int,gpa:double"), data.getSchema(output));
+
+ // sorted data ordered by name
+ assertEquals(tuple((String)null),out.get(0));
+ assertEquals(tuple((String)null),out.get(1));
+ assertEquals(tuple("calvin ellison", 24, 0.71), out.get(2));
+ assertEquals(tuple("wendy johnson", 60, 0.07), out.get(3));
+
+ }
+
private static void deleteDirectory (File path) {
if ( path.exists()) {
File [] files = path.listFiles();
Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_loadavrowithnulls.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_loadavrowithnulls.avro?rev=1489264&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_loadavrowithnulls.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream