You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2013/06/05 16:48:42 UTC
svn commit: r1489894 - in /pig/trunk: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/pi...
Author: rohini
Date: Wed Jun 5 14:48:41 2013
New Revision: 1489894
URL: http://svn.apache.org/r1489894
Log:
PIG-3331: Default values not stored in avro file when using specific schemas during store in AvroStorage (viraj via rohini)
Added:
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testDefaultSchemaWrite.avro (with props)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigSchema2Avro.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1489894&r1=1489893&r2=1489894&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 5 14:48:41 2013
@@ -196,6 +196,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3331: Default values not stored in avro file when using specific schemas during store in AvroStorage (viraj via rohini)
+
PIG-3345: Handle null in DateTime functions (rohini)
PIG-3322: AvroStorage give NPE on reading file with union as top level schema (viraj via rohini)
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigSchema2Avro.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigSchema2Avro.java?rev=1489894&r1=1489893&r2=1489894&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigSchema2Avro.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigSchema2Avro.java Wed Jun 5 14:48:41 2013
@@ -26,6 +26,8 @@ import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.data.DataType;
+import org.codehaus.jackson.JsonNode;
+
/**
* This class contains functions to convert Pig schema to Avro. It consists of
* two sets of methods:
@@ -345,7 +347,10 @@ public class PigSchema2Avro {
/* get doc of output */
String doc = (isPartialSchema) ? pigFields[i].getDescription() : inputField.doc();
- outFields.add(new Field(outname, fieldSchema, doc, null));
+ JsonNode defaultvalue = (inputField != null) ? inputField.defaultValue() : null;
+
+ outFields.add(new Field(outname, fieldSchema, doc, defaultvalue));
+
}
outSchema.setFields(outFields);
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1489894&r1=1489893&r2=1489894&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Wed Jun 5 14:48:41 2013
@@ -627,6 +627,36 @@ public class TestAvroStorage {
}
@Test
+ // Verify the default values specified in the schema in AvroStorage
+ // are actually written to the schema in the output avro file
+ public void testDefaultValueSchemaWrite() throws IOException {
+ String output = outbasedir + "testDefaultValueSchemaWrite";
+ String expected = basedir + "expected_testDefaultSchemaWrite.avro";
+ Data data = resetData(pigServerLocal);
+ data.set("testDefaultValueSchemaWrite",
+ tuple(0,115,115000,115000.1),
+ tuple(1,116,116000,116000.1),
+ tuple(2,117,117000,117000.1),
+ tuple(3,118,118000,118000.1),
+ tuple(4,119,119000,119000.1)
+ );
+ deleteDirectory(new File(output));
+ String [] queries = {
+ " a = LOAD 'testDefaultValueSchemaWrite' USING mock.Storage as " +
+ " (id: int, intval:int, longval:long, floatval:float);",
+ " b = foreach a generate id, longval, floatval;",
+ " c = order b by id;",
+ " STORE c INTO '" + output + "' USING "+
+ " org.apache.pig.piggybank.storage.avro.AvroStorage (' { \"debug\" : 5, \"schema\" : "+
+ " { \"name\" : \"rmyrecord\", \"type\" : \"record\", \"fields\" : [ { \"name\" : \"id\", "+
+ " \"type\" : \"int\" , \"default\" : 0 }, { \"name\" : \"longval\", \"type\" : \"long\","+
+ " \"default\" : 0 }, { \"name\" : \"floatval\", \"type\" : \"float\", \"default\" : 1.0 } ] } } " +
+ " ');" };
+ testAvroStorage(queries);
+ verifyResults(output, expected);
+ }
+
+ @Test
public void testDir() throws IOException {
// Verify that all files in a directory including its sub-directories are loaded.
String output= outbasedir + "testDir";
Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testDefaultSchemaWrite.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testDefaultSchemaWrite.avro?rev=1489894&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testDefaultSchemaWrite.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream