You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/09/27 20:09:15 UTC
svn commit: r1527010 - in /pig/trunk: CHANGES.txt conf/pig.properties
src/org/apache/pig/PigConfiguration.java
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
src/org/apache/pig/tools/pigstats/ScriptState.java
Author: cheolsoo
Date: Fri Sep 27 18:09:14 2013
New Revision: 1527010
URL: http://svn.apache.org/r1527010
Log:
Revert PIG-3484 because it breaks unit tests
Modified:
pig/trunk/CHANGES.txt
pig/trunk/conf/pig.properties
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1527010&r1=1527009&r2=1527010&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Sep 27 18:09:14 2013
@@ -25,14 +25,11 @@ INCOMPATIBLE CHANGES
PIG-3485: Remove CastUtils.bytesToMap(byte[] b) method from LoadCaster interface (cheolsoo)
IMPROVEMENTS
-
-PIG-3484: Make the size of pig.script property configurable (cheolsoo)
OPTIMIZATIONS
BUG FIXES
-
Release 0.12.0 (unreleased changes)
INCOMPATIBLE CHANGES
Modified: pig/trunk/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/trunk/conf/pig.properties?rev=1527010&r1=1527009&r2=1527010&view=diff
==============================================================================
--- pig/trunk/conf/pig.properties (original)
+++ pig/trunk/conf/pig.properties Fri Sep 27 18:09:14 2013
@@ -220,7 +220,3 @@ pig.location.check.strict=false
# Set this option to true if you need to use the old partition filter optimizer.
# Note: Old filter optimizer PColFilterOptimizer will be deprecated in the future.
# pig.exec.useOldPartitionFilterOptimize=true
-
-# By default, the size of pig script stored in job xml is limited to 10,240
-# characters. This property can be used to configure it.
-# pig.script.max.size=<somevalue>
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1527010&r1=1527009&r2=1527010&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Fri Sep 27 18:09:14 2013
@@ -99,14 +99,4 @@ public class PigConfiguration {
* application master getting restarted.
*/
public static final String PIG_OUTPUT_COMMITTER_RECOVERY = "pig.output.committer.recovery.support";
-
- /**
- * This key is used to turn off the inclusion of settings in the jobs.
- */
- public static final String INSERT_ENABLED = "pig.script.info.enabled";
-
- /**
- * Controls the size of Pig script stored in job xml.
- */
- public static final String MAX_SCRIPT_SIZE = "pig.script.max.size";
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1527010&r1=1527009&r2=1527010&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Sep 27 18:09:14 2013
@@ -54,7 +54,6 @@ import org.apache.hadoop.mapred.jobcontr
import org.apache.pig.ComparisonFunc;
import org.apache.pig.ExecType;
import org.apache.pig.LoadFunc;
-import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.executionengine.ExecException;
@@ -105,6 +104,7 @@ import org.apache.pig.impl.util.ObjectSe
import org.apache.pig.impl.util.Pair;
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
+import org.apache.pig.tools.pigstats.ScriptState;
import org.apache.pig.tools.pigstats.mapreduce.MRScriptState;
/**
@@ -436,7 +436,7 @@ public class JobControlCompiler{
Path tmpLocation = null;
// add settings for pig statistics
- String setScriptProp = conf.get(PigConfiguration.INSERT_ENABLED, "true");
+ String setScriptProp = conf.get(ScriptState.INSERT_ENABLED, "true");
if (setScriptProp.equalsIgnoreCase("true")) {
MRScriptState ss = MRScriptState.get();
ss.addSettingsToConf(mro, conf);
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=1527010&r1=1527009&r2=1527010&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Fri Sep 27 18:09:14 2013
@@ -35,7 +35,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.VersionInfo;
import org.apache.pig.ExecType;
-import org.apache.pig.PigConfiguration;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.JarManager;
@@ -127,6 +126,17 @@ public class ScriptState {
MAP_PARTIALAGG;
};
+ /**
+ * Pig property that allows user to turn off the inclusion of settings in
+ * the jobs
+ */
+ public static final String INSERT_ENABLED = "pig.script.info.enabled";
+
+ /**
+ * Restricts the size of Pig script stored in job xml
+ */
+ public static final int MAX_SCRIPT_SIZE = 10240;
+
private static final Log LOG = LogFactory.getLog(ScriptState.class);
private static ThreadLocal<ScriptState> tss = new ThreadLocal<ScriptState>();
@@ -194,10 +204,8 @@ public class ScriptState {
return;
// restrict the size of the script to be stored in job conf
- int maxScriptSize = Integer.valueOf(pigContext.
- getProperties().getProperty(PigConfiguration.MAX_SCRIPT_SIZE, "10240"));
- script = (script.length() > maxScriptSize) ? script.substring(0, maxScriptSize)
- : script;
+ script = (script.length() > MAX_SCRIPT_SIZE) ? script.substring(0,
+ MAX_SCRIPT_SIZE) : script;
// XML parser cann't handle certain characters, including
// the control character (). Use Base64 encoding to