You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2016/07/19 01:26:28 UTC
svn commit: r1753329 - in /pig/trunk: CHANGES.txt conf/pig.properties
src/org/apache/pig/PigConfiguration.java
src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
src/pig-default.properties
Author: daijy
Date: Tue Jul 19 01:26:27 2016
New Revision: 1753329
URL: http://svn.apache.org/viewvc?rev=1753329&view=rev
Log:
PIG-4948: Pig on Tez AM use too much memory on a small cluster
Modified:
pig/trunk/CHANGES.txt
pig/trunk/conf/pig.properties
pig/trunk/src/org/apache/pig/PigConfiguration.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
pig/trunk/src/pig-default.properties
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1753329&r1=1753328&r2=1753329&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jul 19 01:26:27 2016
@@ -60,6 +60,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4948: Pig on Tez AM use too much memory on a small cluster (daijy)
+
PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16 (yangyishan0901m via daijy)
PIG-4950: Fix minor issues with running scripts in non-local FileSystems (petersla via daijy)
Modified: pig/trunk/conf/pig.properties
URL: http://svn.apache.org/viewvc/pig/trunk/conf/pig.properties?rev=1753329&r1=1753328&r2=1753329&view=diff
==============================================================================
--- pig/trunk/conf/pig.properties (original)
+++ pig/trunk/conf/pig.properties Tue Jul 19 01:26:27 2016
@@ -678,3 +678,6 @@ pig.ats.enabled=true
pig.sort.readonce.loadfuncs=org.apache.pig.backend.hadoop.hbase.HBaseStorage,org.apache.pig.backend.hadoop.accumulo.AccumuloStorage
+# If set, Pig will override tez.am.launch.cmd-opts and tez.am.resource.memory.mb to optimal
+# even they are set to a different value. Default value is true.
+#pig.tez.configure.am.memory=false
Modified: pig/trunk/src/org/apache/pig/PigConfiguration.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigConfiguration.java?rev=1753329&r1=1753328&r2=1753329&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigConfiguration.java (original)
+++ pig/trunk/src/org/apache/pig/PigConfiguration.java Tue Jul 19 01:26:27 2016
@@ -434,6 +434,11 @@ public class PigConfiguration {
*/
public static final String ENABLE_ATS = "pig.ats.enabled";
+ /**
+ * Enable ATS for Pig
+ */
+ public static final String PIG_TEZ_CONFIGURE_AM_MEMORY = "pig.tez.configure.am.memory";
+
// Deprecated settings of Pig 0.13
/**
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java?rev=1753329&r1=1753328&r2=1753329&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezSessionManager.java Tue Jul 19 01:26:27 2016
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.pig.PigConfiguration;
import org.apache.pig.backend.hadoop.executionengine.tez.TezJob.TezJobConfig;
import org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper;
import org.apache.pig.impl.PigContext;
@@ -99,7 +100,9 @@ public class TezSessionManager {
MRToTezHelper.translateMRSettingsForTezAM(amConf);
TezScriptState ss = TezScriptState.get();
ss.addDAGSettingsToConf(amConf);
- adjustAMConfig(amConf, tezJobConf);
+ if (amConf.getBoolean(PigConfiguration.PIG_TEZ_CONFIGURE_AM_MEMORY, true)) {
+ adjustAMConfig(amConf, tezJobConf);
+ }
String jobName = amConf.get(PigContext.JOB_NAME, "pig");
TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds);
try {
Modified: pig/trunk/src/pig-default.properties
URL: http://svn.apache.org/viewvc/pig/trunk/src/pig-default.properties?rev=1753329&r1=1753328&r2=1753329&view=diff
==============================================================================
--- pig/trunk/src/pig-default.properties (original)
+++ pig/trunk/src/pig-default.properties Tue Jul 19 01:26:27 2016
@@ -64,3 +64,5 @@ pig.tez.opt.union.unsupported.storefuncs
pig.sort.readonce.loadfuncs=org.apache.pig.backend.hadoop.hbase.HBaseStorage,org.apache.pig.backend.hadoop.accumulo.AccumuloStorage
pig.ats.enabled=true
+
+pig.tez.configure.am.memory=true