You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2010/08/05 19:33:12 UTC
svn commit: r982704 - in /hadoop/pig/trunk:
src/org/apache/pig/builtin/PigStorage.java
test/org/apache/pig/test/TestBZip.java
Author: rding
Date: Thu Aug 5 17:33:11 2010
New Revision: 982704
URL: http://svn.apache.org/viewvc?rev=982704&view=rev
Log:
PIG-1533: Compression codec should be a per-store property
Modified:
hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java
hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=982704&r1=982703&r2=982704&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java Thu Aug 5 17:33:11 2010
@@ -238,6 +238,8 @@ LoadPushDown {
} else if (location.endsWith(".gz")) {
FileOutputFormat.setCompressOutput(job, true);
FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
+ } else {
+ FileOutputFormat.setCompressOutput(job, false);
}
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java?rev=982704&r1=982703&r2=982704&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestBZip.java Thu Aug 5 17:33:11 2010
@@ -17,6 +17,7 @@
*/
package org.apache.pig.test;
+import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
@@ -353,4 +354,31 @@ public class TestBZip {
assertEquals(expectedCount, result);
}
+
+ @Test
+ public void testBzipStoreInMultiQuery() throws Exception {
+ String[] inputData = new String[] {
+ "1\t2\r3\t4"
+ };
+
+ String inputFileName = "input.txt";
+ Util.createInputFile(cluster, inputFileName, inputData);
+
+ PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster
+ .getProperties());
+
+ pig.setBatchOn();
+ pig.registerQuery("a = load '" + inputFileName + "';");
+ pig.registerQuery("store a into 'output.bz2';");
+ pig.registerQuery("store a into 'output';");
+ pig.executeBatch();
+
+ FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
+ pig.getPigContext().getProperties()));
+ FileStatus stat = fs.getFileStatus(new Path("output/part-m-00000"));
+ assertTrue(stat.getLen() > 0);
+
+ stat = fs.getFileStatus(new Path("output.bz2/part-m-00000.bz2"));
+ assertTrue(stat.getLen() > 0);
+ }
}