You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2015/03/05 18:45:56 UTC
svn commit: r1664432 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java
test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc
test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc
Author: daijy
Date: Thu Mar 5 17:45:56 2015
New Revision: 1664432
URL: http://svn.apache.org/r1664432
Log:
PIG-4447: Pig Cannot handle nullable values (arrays and records) in avro records
Added:
pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc
pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1664432&r1=1664431&r2=1664432&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Mar 5 17:45:56 2015
@@ -50,6 +50,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4447: Pig Cannot handle nullable values (arrays and records) in avro records (rdsr via daijy)
+
PIG-4444: Fix unit test failure TestTezAutoParallelism (daijy)
PIG-4445: VALUELIST and VALUESET outputSchema does not match actual schema of data returned when map value schema
Modified: pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java?rev=1664432&r1=1664431&r2=1664432&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/avro/AvroStorageSchemaConversionUtilities.java Thu Mar 5 17:45:56 2015
@@ -222,7 +222,7 @@ public class AvroStorageSchemaConversion
switch(mapAvroSchema.getType()) {
case RECORD:
ResourceSchema innerResourceSchemaRecord =
- avroSchemaToResourceSchema(fieldSchema.getValueType(), schemasInStack,
+ avroSchemaToResourceSchema(mapAvroSchema, schemasInStack,
alreadyDefinedSchemas, allowRecursiveSchema);
mapSchemaFields[0] = new ResourceSchema.ResourceFieldSchema();
mapSchemaFields[0].setType(DataType.TUPLE);
@@ -235,7 +235,7 @@ public class AvroStorageSchemaConversion
case MAP:
case ARRAY:
ResourceSchema innerResourceSchema =
- avroSchemaToResourceSchema(fieldSchema.getValueType(), schemasInStack,
+ avroSchemaToResourceSchema(mapAvroSchema, schemasInStack,
alreadyDefinedSchemas, allowRecursiveSchema);
rf.setSchema(innerResourceSchema);
break;
Added: pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc?rev=1664432&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc (added)
+++ pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableArrayInMap.avsc Thu Mar 5 17:45:56 2015
@@ -0,0 +1,22 @@
+{
+ "name" : "nullableRecordInMap",
+ "namespace" : "org.apache.pig.test.builtin",
+ "type" : "record",
+ "fields" : [
+ {"name" : "key", "type" : "string"},
+ {"name" : "value", "type" : "int"},
+ {
+ "name" : "parameters",
+ "type": {
+ "type": "map",
+ "values": [
+ "null",
+ {
+ "type": "array",
+ "items": "string"
+ }
+ ]
+ }
+ }
+ ]
+}
Added: pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc?rev=1664432&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc (added)
+++ pig/trunk/test/org/apache/pig/builtin/avro/schema/nullableRecordInMap.avsc Thu Mar 5 17:45:56 2015
@@ -0,0 +1,34 @@
+{
+ "name" : "nullableRecordInMap",
+ "namespace" : "org.apache.pig.test.builtin",
+ "type" : "record",
+ "fields" : [
+ {"name" : "key", "type" : "string"},
+ {"name" : "value", "type" : "int"},
+ {
+ "name" : "parameters",
+ "type": [
+ "null",
+ {
+ "type": "map",
+ "values": [
+ "null",
+ {
+ "type": "record",
+ "name": "nullable_record",
+ "fields": [
+ {
+ "name": "id",
+ "type": [
+ "null",
+ "string"
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}