You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/06/27 20:32:21 UTC

svn commit: r1354657 - in /pig/trunk: CHANGES.txt src/org/apache/pig/builtin/PigStorage.java test/org/apache/pig/test/TestPigStorage.java

Author: daijy
Date: Wed Jun 27 18:32:20 2012
New Revision: 1354657

URL: http://svn.apache.org/viewvc?rev=1354657&view=rev
Log:
PIG-2746: Pig doesn't detect all forms of compression extensions properly

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/builtin/PigStorage.java
    pig/trunk/test/org/apache/pig/test/TestPigStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1354657&r1=1354656&r2=1354657&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 27 18:32:20 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
 
 IMPROVEMENTS
 
+PIG-2746: Pig doesn't detect all forms of compression extensions properly (qwertymaniac via daijy)
+
 PIG-2770: Allow easy inclusion of custom build targets (julien)
 
 PIG-2697: pretty print schema via pig.pretty.print.schema (rangadi via jcoveney)

Modified: pig/trunk/src/org/apache/pig/builtin/PigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=1354657&r1=1354656&r2=1354657&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/PigStorage.java Wed Jun 27 18:32:20 2012
@@ -33,9 +33,8 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.BZip2Codec;
 import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
 import org.apache.hadoop.mapreduce.InputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.OutputFormat;
@@ -404,15 +403,13 @@ LoadPushDown, LoadMetadata, StoreMetadat
     }
 
     private void setCompression(Path path, Job job) {
-     	String location=path.getName();
-        if (location.endsWith(".bz2") || location.endsWith(".bz")) {
-            FileOutputFormat.setCompressOutput(job, true);
-            FileOutputFormat.setOutputCompressorClass(job,  BZip2Codec.class);
-        }  else if (location.endsWith(".gz")) {
-            FileOutputFormat.setCompressOutput(job, true);
-            FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
+        CompressionCodecFactory factory = new CompressionCodecFactory(job.getConfiguration());
+        CompressionCodec codec = factory.getCodec(path);
+        if (codec == null) {
+            FileOutputFormat.setCompressOutput(job, false);
         } else {
-            FileOutputFormat.setCompressOutput( job, false);
+            FileOutputFormat.setCompressOutput(job, true);
+            FileOutputFormat.setOutputCompressorClass(job, codec.getClass());
         }
     }
 

Modified: pig/trunk/test/org/apache/pig/test/TestPigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStorage.java?rev=1354657&r1=1354656&r2=1354657&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStorage.java Wed Jun 27 18:32:20 2012
@@ -529,5 +529,28 @@ public class TestPigStorage  {
             String inputFileName = (String)tuple.get(0);
             assertEquals("tagsource value must be part-m-00000", inputFileName, storeFileName);
         }
-    }    
+    }
+
+    @Test
+    public void testPigStorageCompression() throws Exception {
+        final String storeFileName = "part-m-00000.bz2";
+        pigContext.connect();
+
+        String query = "a = LOAD '" + datadir + "' using PigStorage('\\t') " +
+        "as (f1:chararray, f2:int);";
+        pig.registerQuery(query);
+        // Storing in 'aout.bz2' directory will store contents in part-m-00000.bz2
+        // (With BZ2 compression applied automatically)
+        pig.store("a", datadir + "aout.bz2", "PigStorage('\\t', '-schema')");
+
+        // Verify input source tag is present when using -tagsource
+        pig.registerQuery("b = LOAD '" + datadir + "aout.bz2' using PigStorage('\\t', '-tagsource');");
+        pig.registerQuery("c = foreach b generate INPUT_FILE_NAME;");
+        Iterator<Tuple> iter = pig.openIterator("c");
+        while(iter.hasNext()) {
+            Tuple tuple = iter.next();
+            String inputFileName = (String)tuple.get(0);
+            assertEquals("tagsource value must be part-m-00000.bz2", inputFileName, storeFileName);
+        }
+    }
 }