You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/03/20 03:32:47 UTC
svn commit: r1302757 - in /pig/trunk: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/pi...
Author: daijy
Date: Tue Mar 20 02:32:46 2012
New Revision: 1302757
URL: http://svn.apache.org/viewvc?rev=1302757&view=rev
Log:
PIG-2505: AvroStorage won't read any file not ending in .avro
Added:
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testFileWithNoExtension.avro (with props)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroInputFormat.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1302757&r1=1302756&r2=1302757&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 20 02:32:46 2012
@@ -283,6 +283,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2505: AvroStorage won't read any file not ending in .avro (russell.jurney via daijy)
+
PIG-2585: Enable ignored e2e test cases (daijy)
PIG-2563: IndexOutOfBoundsException: while projecting fields from a bag (daijy)
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroInputFormat.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroInputFormat.java?rev=1302757&r1=1302756&r2=1302757&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroInputFormat.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/PigAvroInputFormat.java Tue Mar 20 02:32:46 2012
@@ -54,18 +54,6 @@ public class PigAvroInputFormat extends
}
/**
- * Ignore files not ending with ".avro"
- */
- @Override
- protected List<FileStatus> listStatus(JobContext context) throws IOException {
- List<FileStatus> result = new ArrayList<FileStatus>();
- for (FileStatus file : super.listStatus(context))
- if (file.getPath().getName().endsWith(PigAvroOutputFormat.EXT))
- result.add(file);
- return result;
- }
-
- /**
* Create and return an avro record reader.
* It uses the input schema passed in to the
* constructor.
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1302757&r1=1302756&r2=1302757&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Tue Mar 20 02:32:46 2012
@@ -69,6 +69,7 @@ public class TestAvroStorage {
final private String testRecordSchema = getInputFile("test_record.avsc");
final private String testTextFile = getInputFile("test_record.txt");
final private String testSingleTupleBagFile = getInputFile("messages.avro");
+ final private String testNoExtensionFile = getInputFile("test_no_extension");
@BeforeClass
public static void setup() throws ExecException {
@@ -307,6 +308,27 @@ public class TestAvroStorage {
};
testAvroStorage( queries);
}
+
+ @Test
+ public void testFileWithNoExtension() throws IOException {
+ String output= outbasedir + "testFileWithNoExtension";
+ String expected = basedir + "expected_testFileWithNoExtension.avro";
+ deleteDirectory(new File(output));
+ String [] queries = {
+ " avro = LOAD '" + testNoExtensionFile + " ' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
+ " avro1 = FILTER avro BY member_id > 1211;",
+ " avro2 = FOREACH avro1 GENERATE member_id, browser_id, tracking_time, act_content ;",
+ " STORE avro2 INTO '" + output + "' " +
+ " USING org.apache.pig.piggybank.storage.avro.AvroStorage (" +
+ "'{\"data\": \"" + testNoExtensionFile + "\" ," +
+ " \"field0\": \"int\", " +
+ " \"field1\": \"def:browser_id\", " +
+ " \"field3\": \"def:act_content\" " +
+ " }');"
+ };
+ testAvroStorage( queries);
+ verifyResults(output, expected);
+ }
private static void deleteDirectory (File path) {
if ( path.exists()) {
Added: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testFileWithNoExtension.avro
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testFileWithNoExtension.avro?rev=1302757&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/avro_test_files/expected_testFileWithNoExtension.avro
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream