You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2012/12/10 07:10:59 UTC
svn commit: r1419206 - in /pig/trunk: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/pi...
Author: cheolsoo
Date: Mon Dec 10 06:10:58 2012
New Revision: 1419206
URL: http://svn.apache.org/viewvc?rev=1419206&view=rev
Log:
PIG-3075: Allow AvroStorage STORE Operations To Use Schema Specified By URI (nwhite via cheolsoo)
Added:
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro (with props)
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1419206&r1=1419205&r2=1419206&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Dec 10 06:10:58 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-3075: Allow AvroStorage STORE Operations To Use Schema Specified By URI (nwhite via cheolsoo)
+
PIG-3062: Change HBaseStorage to permit overriding pushProjection (billgraham)
PIG-3016: Modernize more tests (jcoveney via cheolsoo)
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java?rev=1419206&r1=1419205&r2=1419206&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java Mon Dec 10 06:10:58 2012
@@ -477,6 +477,7 @@ public class AvroStorage extends FileInp
|| name.equalsIgnoreCase("same")
|| name.equalsIgnoreCase("schema")
|| name.equalsIgnoreCase("schema_file")
+ || name.equalsIgnoreCase("schema_uri")
|| name.matches("field\\d+")) {
/* store value as string */
map.put(name, value);
@@ -541,6 +542,12 @@ public class AvroStorage extends FileInp
nullable = (Boolean) value;
} else if (name.equalsIgnoreCase("schema")) {
outputAvroSchema = Schema.parse((String) value);
+ } else if (name.equalsIgnoreCase("schema_uri")) {
+ /* use the contents of the specified path as output schema */
+ Path path = new Path( ((String) value).trim());
+ AvroStorageLog.details("schema_uri path=" + path.toUri().toString());
+ FileSystem fs = FileSystem.get(path.toUri(), new Configuration());
+ outputAvroSchema = getSchemaFromFile(path, fs);
} else if (name.matches("field\\d+")) {
/*set schema of dth field */
if (fields == null)
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1419206&r1=1419205&r2=1419206&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Mon Dec 10 06:10:58 2012
@@ -80,6 +80,7 @@ public class TestAvroStorage {
final private String testDir21AllFiles = getInputFile("{test_dir2,test_dir1}/test_glob*.avro");
final private String testNoMatchedFiles = getInputFile("test_dir{1,2}/file_that_does_not_exist*.avro");
final private String testArrayFile = getInputFile("test_array.avro");
+ final private String testArraySchema = getInputFile("test_array.avsc");
final private String testRecordFile = getInputFile("test_record.avro");
final private String testRecordSchema = getInputFile("test_record.avsc");
final private String testGenericUnionFile = getInputFile("test_generic_union.avro");
@@ -687,17 +688,31 @@ public class TestAvroStorage {
verifyResults(output, expected);
}
-
@Test
public void testArrayWithSchema() throws IOException {
String output= outbasedir + "testArrayWithSchema";
String expected = basedir + "expected_testArrayWithSchema.avro";
deleteDirectory(new File(output));
String [] queries = {
+ " in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " STORE in INTO '" + output +
+ "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ( " +
+ " 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
+ };
+ testAvroStorage( queries);
+ verifyResults(output, expected);
+ }
+
+ @Test
+ public void testArrayWithSchemaURI() throws IOException {
+ String output= outbasedir + "testArrayWithSchemaURI";
+ String expected = basedir + "expected_testArrayWithSchemaURI.avro"; // doubles (not floats) stored
+ deleteDirectory(new File(output));
+ String [] queries = {
" in = LOAD '" + testArrayFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
" STORE in INTO '" + output +
"' USING org.apache.pig.piggybank.storage.avro.AvroStorage ( " +
- " 'schema', '{\"type\":\"array\",\"items\":\"float\"}' );"
+ " 'schema_uri', '" + testArraySchema + "' );"
};
testAvroStorage( queries);
verifyResults(output, expected);
Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro?rev=1419206&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro
------------------------------------------------------------------------------
svn:executable = *
Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testArrayWithSchemaURI.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc?rev=1419206&view=auto
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc (added)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/test_array.avsc Mon Dec 10 06:10:58 2012
@@ -0,0 +1 @@
+{"type":"array","items":"double"}
\ No newline at end of file