You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/01/31 21:03:27 UTC
svn commit: r1563236 - in /pig/trunk: ./ conf/ ivy/ src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
test/org/apache/pig/test/
Author: daijy
Date: Fri Jan 31 20:03:26 2014
New Revision: 1563236
URL: http://svn.apache.org/r1563236
Log:
Revert PIG-3299: Provide support for LazyOutputFormat to avoid creating empty files
Modified:
pig/trunk/CHANGES.txt
pig/trunk/conf/pig.properties
pig/trunk/ivy/libraries.properties
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/test/org/apache/pig/test/TestStore.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1563236&r1=1563235&r2=1563236&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Jan 31 20:03:26 2014
@@ -30,8 +30,6 @@ PIG-2207: Support custom counters for ag
IMPROVEMENTS
-PIG-3299: Provide support for LazyOutputFormat to avoid creating empty files (lbendig via daijy)
-
PIG-3730: Performance issue in SelfSpillBag (rajesh.balamohan via rohini)
PIG-3654: Add class cache to PigContext (tmwoodruff via daijy)
Modified: pig/trunk/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/trunk/conf/pig.properties?rev=1563236&r1=1563235&r2=1563236&view=diff
==============================================================================
--- pig/trunk/conf/pig.properties (original)
+++ pig/trunk/conf/pig.properties Fri Jan 31 20:03:26 2014
@@ -242,7 +242,3 @@ pig.location.check.strict=false
# Set value in long as a threshold number of bytes to convert
# jobs with smaller input data size to run in local mode
# pig.auto.local.input.maxbytes=100000000
-
-# When enabled, jobs won't create empty part files if no output is written. In this case
-# PigOutputFormat will be wrapped with org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat.
-# pig.output.lazy=true
Modified: pig/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/trunk/ivy/libraries.properties?rev=1563236&r1=1563235&r2=1563236&view=diff
==============================================================================
--- pig/trunk/ivy/libraries.properties (original)
+++ pig/trunk/ivy/libraries.properties Fri Jan 31 20:03:26 2014
@@ -37,8 +37,8 @@ jasper.version=6.1.14
groovy.version=1.8.6
guava.version=11.0
jersey-core.version=1.8
-hadoop-core.version=1.0.4
-hadoop-test.version=1.0.4
+hadoop-core.version=1.0.0
+hadoop-test.version=1.0.0
hadoop-common.version=2.0.3-alpha
hadoop-hdfs.version=2.0.3-alpha
hadoop-mapreduce.version=2.0.3-alpha
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1563236&r1=1563235&r2=1563236&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Fri Jan 31 20:03:26 2014
@@ -148,12 +148,5 @@ public class PigConfiguration {
* Controls the max threshold size to convert jobs to run in local mode
*/
public static final String PIG_AUTO_LOCAL_INPUT_MAXBYTES = "pig.auto.local.input.maxbytes";
-
- /**
- * This key is used to define whether PigOutputFormat will be wrapped with LazyOutputFormat
- * so that jobs won't write empty part files if no output is generated
- */
- public static final String PIG_OUTPUT_LAZY = "pig.output.lazy";
-
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1563236&r1=1563235&r2=1563236&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Jan 31 20:03:26 2014
@@ -54,7 +54,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobPriority;
import org.apache.hadoop.mapred.jobcontrol.Job;
import org.apache.hadoop.mapred.jobcontrol.JobControl;
-import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.pig.ComparisonFunc;
import org.apache.pig.ExecType;
import org.apache.pig.LoadFunc;
@@ -669,14 +668,8 @@ public class JobControlCompiler{
sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
}
- // the OutputFormat we report to Hadoop is always PigOutputFormat which
- // can be wrapped with LazyOutputFormat if PigConfiguration.PIG_OUTPUT_LAZY is set
- if ("true".equalsIgnoreCase(conf.get(PigConfiguration.PIG_OUTPUT_LAZY))) {
- LazyOutputFormat.setOutputFormatClass(nwJob, PigOutputFormat.class);
- }
- else {
- nwJob.setOutputFormatClass(PigOutputFormat.class);
- }
+ // the OutputFormat we report to Hadoop is always PigOutputFormat
+ nwJob.setOutputFormatClass(PigOutputFormat.class);
if (mapStores.size() + reduceStores.size() == 1) { // single store case
log.info("Setting up single store job");
@@ -713,6 +706,8 @@ public class JobControlCompiler{
.getTemporaryPath(pigContext).toString();
tmpLocation = new Path(tmpLocationStr);
+ nwJob.setOutputFormatClass(PigOutputFormat.class);
+
boolean disableCounter = conf.getBoolean("pig.disable.counter", false);
if (disableCounter) {
log.info("Disable Pig custom output counters");
Modified: pig/trunk/test/org/apache/pig/test/TestStore.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=1563236&r1=1563235&r2=1563236&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestStore.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestStore.java Fri Jan 31 20:03:26 2014
@@ -19,7 +19,6 @@ package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.io.BufferedReader;
@@ -46,7 +45,6 @@ import org.apache.hadoop.mapreduce.TaskA
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.pig.EvalFunc;
import org.apache.pig.ExecType;
-import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.PigServer;
import org.apache.pig.ResourceSchema;
@@ -693,71 +691,6 @@ public class TestStore {
Util.deleteFile(ps.getPigContext(), TESTDIR);
}
}
-
- /**
- * Test whether "part-m-00000" file is created on empty output when
- * {@link PigConfiguration#PIG_OUTPUT_LAZY} is set.
- * The test covers multi store and single store case in local and mapreduce mode
- *
- * @throws IOException
- */
- @Test
- public void testEmptyPartFileCreation() throws IOException {
- PigServer ps = null;
-
- try {
- ExecType[] modes = new ExecType[] { ExecType.LOCAL, ExecType.MAPREDUCE};
- String[] inputData = new String[]{"hello\tworld", "hi\tworld", "bye\tworld"};
-
- String multiStoreScript = "a = load '"+ inputFileName + "';" +
- "b = filter a by $0 == 'hey';" +
- "c = filter a by $1 == 'globe';" +
- "d = limit a 2;" +
- "e = foreach d generate *, 'x';" +
- "f = filter e by $3 == 'y';" +
- "store b into '" + outputFileName + "_1';" +
- "store c into '" + outputFileName + "_2';" +
- "store f into '" + outputFileName + "_3';";
-
- String singleStoreScript = "a = load '"+ inputFileName + "';" +
- "b = filter a by $0 == 'hey';" +
- "store b into '" + outputFileName + "_1';" ;
-
- for (ExecType execType : modes) {
- for(boolean isMultiStore: new boolean[] { true, false}) {
- String script = (isMultiStore ? multiStoreScript :
- singleStoreScript);
- // since we will be switching between map red and local modes
- // we will need to make sure filelocalizer is reset before each
- // run.
- FileLocalizer.setInitialized(false);
- if(execType == ExecType.MAPREDUCE) {
- ps = new PigServer(ExecType.MAPREDUCE,
- cluster.getProperties());
- } else {
- Properties props = new Properties();
- props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
- ps = new PigServer(ExecType.LOCAL, props);
- }
- ps.getPigContext().getProperties().setProperty(
- PigConfiguration.PIG_OUTPUT_LAZY, "true");
- Util.deleteFile(ps.getPigContext(), TESTDIR);
- ps.setBatchOn();
- Util.createInputFile(ps.getPigContext(),
- inputFileName, inputData);
- Util.registerMultiLineQuery(ps, script);
- ps.executeBatch();
- for(int i = 1; i <= (isMultiStore ? 3 : 1); i++) {
- String output = "part-m-00000";
- assertFalse("For an empty output part-m-00000 should not exist in " +
- execType + " mode", Util.exists(ps.getPigContext(), output));
- }
- }
- }
- } finally {
- Util.deleteFile(ps.getPigContext(), TESTDIR);
- }
- }
// A UDF which always throws an Exception so that the job can fail
public static class FailUDF extends EvalFunc<String> {