You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/03/28 17:13:52 UTC
svn commit: r1582793 - in /pig/branches/branch-0.12: ./
src/org/apache/pig/builtin/ test/org/apache/pig/builtin/
test/org/apache/pig/builtin/avro/code/pig/
test/org/apache/pig/builtin/avro/data/json/
test/org/apache/pig/builtin/avro/schema/
Author: cheolsoo
Date: Fri Mar 28 16:13:51 2014
New Revision: 1582793
URL: http://svn.apache.org/r1582793
Log:
PIG-3833: Relation loaded by AvroStorage with schema is projected incorrectly
in foreach statement (jeongjinku via cheolsoo)
Added:
pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
Modified:
pig/branches/branch-0.12/CHANGES.txt
pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java
pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java
Modified: pig/branches/branch-0.12/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/CHANGES.txt?rev=1582793&r1=1582792&r2=1582793&view=diff
==============================================================================
--- pig/branches/branch-0.12/CHANGES.txt (original)
+++ pig/branches/branch-0.12/CHANGES.txt Fri Mar 28 16:13:51 2014
@@ -34,6 +34,8 @@ PIG-3480: TFile-based tmpfile compressio
BUG FIXES
+PIG-3833: Relation loaded by AvroStorage with schema is projected incorrectly in foreach statement (jeongjinku via cheolsoo)
+
PIG-3794: pig -useHCatalog fails using pig command line interface on HDInsight (ehans via daijy)
PIG-3827: Custom partitioner is not picked up with secondary sort optimization (daijy)
Modified: pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java?rev=1582793&r1=1582792&r2=1582793&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/builtin/AvroStorage.java Fri Mar 28 16:13:51 2014
@@ -210,6 +210,7 @@ public class AvroStorage extends LoadFun
public final void setUDFContextSignature(final String signature) {
udfContextSignature = signature;
super.setUDFContextSignature(signature);
+ updateSchemaFromInputAvroSchema();
}
/**
@@ -620,16 +621,24 @@ public class AvroStorage extends LoadFun
*/
public final Schema getInputAvroSchema() {
if (schema == null) {
- String schemaString = getProperties().getProperty(INPUT_AVRO_SCHEMA);
- if (schemaString != null) {
- Schema s = new Schema.Parser().parse(schemaString);
- schema = s;
- }
+ updateSchemaFromInputAvroSchema();
}
return schema;
}
- /*
+ /**
+ * Utility function that gets the input avro schema from the udf
+ * properties and updates schema for this instance.
+ */
+ private final void updateSchemaFromInputAvroSchema() {
+ String schemaString = getProperties().getProperty(INPUT_AVRO_SCHEMA);
+ if (schemaString != null) {
+ Schema s = new Schema.Parser().parse(schemaString);
+ schema = s;
+ }
+ }
+
+ /**
* @see org.apache.pig.LoadFunc#getInputFormat()
*/
@Override
Modified: pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java?rev=1582793&r1=1582792&r2=1582793&view=diff
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java (original)
+++ pig/branches/branch-0.12/test/org/apache/pig/builtin/TestAvroStorage.java Fri Mar 28 16:13:51 2014
@@ -100,6 +100,7 @@ public class TestAvroStorage {
"recordWithRepeatedSubRecords",
"recursiveRecord",
"projectionTest",
+ "projectionTestWithSchema",
"recordsWithSimpleUnion",
"recordsWithSimpleUnionOutput",
};
@@ -332,9 +333,8 @@ public class TestAvroStorage {
"OUTFILE", createOutputName())
);
verifyResults(createOutputName(),check);
- }
-
-
+ }
+
@Test public void testProjection() throws Exception {
final String input = basedir + "data/avro/uncompressed/records.avro";
final String check = basedir + "data/avro/uncompressed/projectionTest.avro";
@@ -346,9 +346,22 @@ public class TestAvroStorage {
"OUTFILE", createOutputName())
);
verifyResults(createOutputName(),check);
- }
+ }
+
+ @Test public void testProjectionWithSchema() throws Exception {
+ final String input = basedir + "data/avro/uncompressed/records.avro";
+ final String check = basedir + "data/avro/uncompressed/projectionTestWithSchema.avro";
+ testAvroStorage(true, basedir + "code/pig/projection_test_with_schema.pig",
+ ImmutableMap.of(
+ "INFILE", input,
+ "AVROSTORAGE_IN_2", "-f " + basedir + "schema/records.avsc",
+ "AVROSTORAGE_OUT_1", "projectionTest",
+ "AVROSTORAGE_OUT_2", "-n org.apache.pig.test.builtin",
+ "OUTFILE", createOutputName())
+ );
+ verifyResults(createOutputName(),check);
+ }
-
@Test public void testDates() throws Exception {
final String input = basedir + "data/avro/uncompressed/records.avro";
final String check = basedir + "data/avro/uncompressed/recordsAsOutputByPigWithDates.avro";
@@ -762,7 +775,6 @@ public class TestAvroStorage {
pigServerLocal.registerQuery("B = LOAD '" + createOutputName() + "' USING AvroStorage();");
pigServerLocal.registerQuery("C = FOREACH B generate maps#'key1';");
pigServerLocal.registerQuery("STORE C INTO 'out' USING mock.Storage();");
-
List<Tuple> out = data.get("out");
assertEquals(tuple("v11"), out.get(0));
@@ -836,7 +848,7 @@ public class TestAvroStorage {
assertEquals(expected.size(), count);
}
}
- }
+ }
private Set<GenericData.Record> getExpected (String pathstr ) throws IOException {
@@ -874,7 +886,7 @@ public class TestAvroStorage {
}
}
return ret;
- }
+ }
}
Added: pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig?rev=1582793&view=auto
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig (added)
+++ pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/code/pig/projection_test_with_schema.pig Fri Mar 28 16:13:51 2014
@@ -0,0 +1,4 @@
+in = LOAD '$INFILE' USING AvroStorage('','$AVROSTORAGE_IN_2');
+out = FOREACH in GENERATE $0, $1, $3;
+RMF $OUTFILE;
+STORE out INTO '$OUTFILE' USING AvroStorage('$AVROSTORAGE_OUT_1','$AVROSTORAGE_OUT_2');
\ No newline at end of file
Added: pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json?rev=1582793&view=auto
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json (added)
+++ pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/data/json/projectionTestWithSchema.json Fri Mar 28 16:13:51 2014
@@ -0,0 +1,15 @@
+{
+ "key" : "A",
+ "intValue" : 1,
+ "booleanValue" : true
+}
+{
+ "key" : "B",
+ "intValue" : 2,
+ "booleanValue" : true
+}
+{
+ "key" : "C",
+ "intValue" : 3,
+ "booleanValue" : false
+}
\ No newline at end of file
Added: pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc?rev=1582793&view=auto
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc (added)
+++ pig/branches/branch-0.12/test/org/apache/pig/builtin/avro/schema/projectionTestWithSchema.avsc Fri Mar 28 16:13:51 2014
@@ -0,0 +1,10 @@
+{
+ "name" : "projectionTestWithSchema",
+ "namespace" : "org.apache.pig.test.builtin",
+ "type" : "record",
+ "fields" : [
+ {"name" : "key", "type" : "string"},
+ {"name" : "intValue", "type" : "int"},
+ {"name" : "booleanValue", "type" : "boolean"}
+ ]
+}
\ No newline at end of file