You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/04/25 23:42:52 UTC
svn commit: r1096617 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/PigServer.java
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
test/org/apache/pig/test/TestBZip.java
Author: daijy
Date: Mon Apr 25 21:42:52 2011
New Revision: 1096617
URL: http://svn.apache.org/viewvc?rev=1096617&view=rev
Log:
PIG-1814: mapred.output.compress in SET statement does not work
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1096617&r1=1096616&r2=1096617&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Mon Apr 25 21:42:52 2011
@@ -170,6 +170,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1814: mapred.output.compress in SET statement does not work (daijy)
+
PIG-1976: One more TwoLevelAccess to remove (daijy)
PIG-1865: BinStorage/PigStorageSchema cannot load data from a different namenode (daijy)
Modified: pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/PigServer.java?rev=1096617&r1=1096616&r2=1096617&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/PigServer.java Mon Apr 25 21:42:52 2011
@@ -242,16 +242,6 @@ public class PigServer {
pigContext.connect();
}
- if( "true".equals( pigContext.getProperties().getProperty( "mapred.output.compress" ) ) ) {
- pigContext.getProperties().setProperty( "output.compression.enabled", "true" );
- String codec = pigContext.getProperties().getProperty( "mapred.output.compression.codec" );
- if( codec == null ) {
- throw new RuntimeException( "'mapred.output.compress' is set but no value is specified for 'mapred.output.compression.codec'." );
- } else {
- pigContext.getProperties().setProperty( "output.compression.codec", codec );
- }
- }
-
addJarsFromProperties();
}
Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1096617&r1=1096616&r2=1096617&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Mon Apr 25 21:42:52 2011
@@ -356,6 +356,17 @@ public class JobControlCompiler{
}else{
log.info("mapred.job.reduce.markreset.buffer.percent is set to " + conf.get("mapred.job.reduce.markreset.buffer.percent"));
}
+
+ // Convert mapred.output.* to output.compression.*, See PIG-1791
+ if( "true".equals( conf.get( "mapred.output.compress" ) ) ) {
+ conf.set( "output.compression.enabled", "true" );
+ String codec = conf.get( "mapred.output.compression.codec" );
+ if( codec == null ) {
+ throw new JobCreationException("'mapred.output.compress' is set but no value is specified for 'mapred.output.compression.codec'." );
+ } else {
+ conf.set( "output.compression.codec", codec );
+ }
+ }
try{
//Process the POLoads
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java?rev=1096617&r1=1096616&r2=1096617&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java Mon Apr 25 21:42:52 2011
@@ -18,16 +18,16 @@
package org.apache.pig.test;
import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Properties;
@@ -53,6 +53,7 @@ import org.junit.AfterClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+
@RunWith(JUnit4.class)
public class TestBZip {
static MiniCluster cluster = MiniCluster.buildCluster();
@@ -578,4 +579,40 @@ public class TestBZip {
out.close();
}
+ // See PIG-1714
+ @Test
+ public void testBzipStoreInMultiQuery3() throws Exception {
+ String[] inputData = new String[] {
+ "1\t2\r3\t4"
+ };
+
+ String inputFileName = "input3.txt";
+ Util.createInputFile(cluster, inputFileName, inputData);
+
+ String inputScript = "set mapred.output.compress true\n" +
+ "set mapred.output.compression.codec org.apache.hadoop.io.compress.BZip2Codec\n" +
+ "a = load '" + inputFileName + "';\n" +
+ "store a into 'output3.bz2';\n" +
+ "store a into 'output3';";
+
+ String inputScriptName = "script3.txt";
+ PrintWriter pw = new PrintWriter(new FileWriter(inputScriptName));
+ pw.println(inputScript);
+ pw.close();
+
+ PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster
+ .getProperties());
+
+ FileInputStream fis = new FileInputStream(inputScriptName);
+ pig.registerScript(fis);
+
+ FileSystem fs = FileSystem.get(ConfigurationUtil.toConfiguration(
+ pig.getPigContext().getProperties()));
+ FileStatus stat = fs.getFileStatus(new Path("output3/part-m-00000.bz2"));
+ assertTrue(stat.getLen() > 0);
+
+ stat = fs.getFileStatus(new Path("output3.bz2/part-m-00000.bz2"));
+ assertTrue(stat.getLen() > 0);
+ }
+
}