You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2013/09/30 18:26:16 UTC
svn commit: r1527631 - in /pig/trunk: CHANGES.txt conf/pig.properties
src/org/apache/pig/PigConfiguration.java
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
src/org/apache/pig/tools/pigstats/ScriptState.java
Author: cheolsoo
Date: Mon Sep 30 16:26:15 2013
New Revision: 1527631
URL: http://svn.apache.org/r1527631
Log:
PIG-3484: Make the size of pig.script property configurable (cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/conf/pig.properties
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1527631&r1=1527630&r2=1527631&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Sep 30 16:26:15 2013
@@ -25,11 +25,14 @@ INCOMPATIBLE CHANGES
PIG-3485: Remove CastUtils.bytesToMap(byte[] b) method from LoadCaster interface (cheolsoo)
IMPROVEMENTS
+
+PIG-3484: Make the size of pig.script property configurable (cheolsoo)
OPTIMIZATIONS
BUG FIXES
+
Release 0.12.0 (unreleased changes)
INCOMPATIBLE CHANGES
Modified: pig/trunk/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/trunk/conf/pig.properties?rev=1527631&r1=1527630&r2=1527631&view=diff
==============================================================================
--- pig/trunk/conf/pig.properties (original)
+++ pig/trunk/conf/pig.properties Mon Sep 30 16:26:15 2013
@@ -220,3 +220,7 @@ pig.location.check.strict=false
# Set this option to true if you need to use the old partition filter optimizer.
# Note: Old filter optimizer PColFilterOptimizer will be deprecated in the future.
# pig.exec.useOldPartitionFilterOptimize=true
+
+# By default, the size of pig script stored in job xml is limited to 10,240
+# characters. This property can be used to configure it.
+# pig.script.max.size=<somevalue>
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1527631&r1=1527630&r2=1527631&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Mon Sep 30 16:26:15 2013
@@ -99,4 +99,14 @@ public class PigConfiguration {
* application master getting restarted.
*/
public static final String PIG_OUTPUT_COMMITTER_RECOVERY = "pig.output.committer.recovery.support";
+
+ /**
+ * This key is used to turn off the inclusion of settings in the jobs.
+ */
+ public static final String INSERT_ENABLED = "pig.script.info.enabled";
+
+ /**
+ * Controls the size of Pig script stored in job xml.
+ */
+ public static final String MAX_SCRIPT_SIZE = "pig.script.max.size";
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1527631&r1=1527630&r2=1527631&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Mon Sep 30 16:26:15 2013
@@ -54,6 +54,7 @@ import org.apache.hadoop.mapred.jobcontr
import org.apache.pig.ComparisonFunc;
import org.apache.pig.ExecType;
import org.apache.pig.LoadFunc;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.StoreFuncInterface;
import org.apache.pig.backend.executionengine.ExecException;
@@ -104,7 +105,6 @@ import org.apache.pig.impl.util.ObjectSe
import org.apache.pig.impl.util.Pair;
import org.apache.pig.impl.util.UDFContext;
import org.apache.pig.impl.util.Utils;
-import org.apache.pig.tools.pigstats.ScriptState;
import org.apache.pig.tools.pigstats.mapreduce.MRScriptState;
/**
@@ -436,7 +436,7 @@ public class JobControlCompiler{
Path tmpLocation = null;
// add settings for pig statistics
- String setScriptProp = conf.get(ScriptState.INSERT_ENABLED, "true");
+ String setScriptProp = conf.get(PigConfiguration.INSERT_ENABLED, "true");
if (setScriptProp.equalsIgnoreCase("true")) {
MRScriptState ss = MRScriptState.get();
ss.addSettingsToConf(mro, conf);
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=1527631&r1=1527630&r2=1527631&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Mon Sep 30 16:26:15 2013
@@ -35,6 +35,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.VersionInfo;
import org.apache.pig.ExecType;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.JarManager;
@@ -126,17 +127,6 @@ public class ScriptState {
MAP_PARTIALAGG;
};
- /**
- * Pig property that allows user to turn off the inclusion of settings in
- * the jobs
- */
- public static final String INSERT_ENABLED = "pig.script.info.enabled";
-
- /**
- * Restricts the size of Pig script stored in job xml
- */
- public static final int MAX_SCRIPT_SIZE = 10240;
-
private static final Log LOG = LogFactory.getLog(ScriptState.class);
private static ThreadLocal<ScriptState> tss = new ThreadLocal<ScriptState>();
@@ -204,8 +194,15 @@ public class ScriptState {
return;
// restrict the size of the script to be stored in job conf
- script = (script.length() > MAX_SCRIPT_SIZE) ? script.substring(0,
- MAX_SCRIPT_SIZE) : script;
+ int maxScriptSize = 10240;
+ if (pigContext != null) {
+ String prop = pigContext.getProperties().getProperty(PigConfiguration.MAX_SCRIPT_SIZE);
+ if (prop != null) {
+ maxScriptSize = Integer.valueOf(prop);
+ }
+ }
+ script = (script.length() > maxScriptSize) ? script.substring(0, maxScriptSize)
+ : script;
// XML parser cann't handle certain characters, including
// the control character (). Use Base64 encoding to