You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/06/27 20:32:21 UTC
svn commit: r1354657 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/builtin/PigStorage.java
test/org/apache/pig/test/TestPigStorage.java
Author: daijy
Date: Wed Jun 27 18:32:20 2012
New Revision: 1354657
URL: http://svn.apache.org/viewvc?rev=1354657&view=rev
Log:
PIG-2746: Pig doesn't detect all forms of compression extensions properly
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/builtin/PigStorage.java
pig/trunk/test/org/apache/pig/test/TestPigStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1354657&r1=1354656&r2=1354657&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jun 27 18:32:20 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2746: Pig doesn't detect all forms of compression extensions properly (qwertymaniac via daijy)
+
PIG-2770: Allow easy inclusion of custom build targets (julien)
PIG-2697: pretty print schema via pig.pretty.print.schema (rangadi via jcoveney)
Modified: pig/trunk/src/org/apache/pig/builtin/PigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=1354657&r1=1354656&r2=1354657&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/PigStorage.java Wed Jun 27 18:32:20 2012
@@ -33,9 +33,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.compress.BZip2Codec;
import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.OutputFormat;
@@ -404,15 +403,13 @@ LoadPushDown, LoadMetadata, StoreMetadat
}
private void setCompression(Path path, Job job) {
- String location=path.getName();
- if (location.endsWith(".bz2") || location.endsWith(".bz")) {
- FileOutputFormat.setCompressOutput(job, true);
- FileOutputFormat.setOutputCompressorClass(job, BZip2Codec.class);
- } else if (location.endsWith(".gz")) {
- FileOutputFormat.setCompressOutput(job, true);
- FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
+ CompressionCodecFactory factory = new CompressionCodecFactory(job.getConfiguration());
+ CompressionCodec codec = factory.getCodec(path);
+ if (codec == null) {
+ FileOutputFormat.setCompressOutput(job, false);
} else {
- FileOutputFormat.setCompressOutput( job, false);
+ FileOutputFormat.setCompressOutput(job, true);
+ FileOutputFormat.setOutputCompressorClass(job, codec.getClass());
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestPigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStorage.java?rev=1354657&r1=1354656&r2=1354657&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStorage.java Wed Jun 27 18:32:20 2012
@@ -529,5 +529,28 @@ public class TestPigStorage {
String inputFileName = (String)tuple.get(0);
assertEquals("tagsource value must be part-m-00000", inputFileName, storeFileName);
}
- }
+ }
+
+ @Test
+ public void testPigStorageCompression() throws Exception {
+ final String storeFileName = "part-m-00000.bz2";
+ pigContext.connect();
+
+ String query = "a = LOAD '" + datadir + "' using PigStorage('\\t') " +
+ "as (f1:chararray, f2:int);";
+ pig.registerQuery(query);
+ // Storing in 'aout.bz2' directory will store contents in part-m-00000.bz2
+ // (With BZ2 compression applied automatically)
+ pig.store("a", datadir + "aout.bz2", "PigStorage('\\t', '-schema')");
+
+ // Verify input source tag is present when using -tagsource
+ pig.registerQuery("b = LOAD '" + datadir + "aout.bz2' using PigStorage('\\t', '-tagsource');");
+ pig.registerQuery("c = foreach b generate INPUT_FILE_NAME;");
+ Iterator<Tuple> iter = pig.openIterator("c");
+ while(iter.hasNext()) {
+ Tuple tuple = iter.next();
+ String inputFileName = (String)tuple.get(0);
+ assertEquals("tagsource value must be part-m-00000.bz2", inputFileName, storeFileName);
+ }
+ }
}