You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2015/05/02 00:43:14 UTC
svn commit: r1677248 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
Author: daijy
Date: Fri May 1 22:43:13 2015
New Revision: 1677248
URL: http://svn.apache.org/r1677248
Log:
PIG-4529: Pig on tez hit counter limit imposed by MR
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1677248&r1=1677247&r2=1677248&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 1 22:43:13 2015
@@ -76,6 +76,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4529: Pig on tez hit counter limit imposed by MR (daijy)
+
PIG-4524: Pig Minicluster unit tests broken by TEZ-2333 (daijy)
PIG-4527: NON-ASCII Characters in Javadoc break 'ant docs' (nielsbasjes via daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java?rev=1677248&r1=1677247&r2=1677248&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java Fri May 1 22:43:13 2015
@@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.pig.PigConfiguration;
import org.apache.pig.PigException;
import org.apache.pig.PigWarning;
@@ -116,6 +117,18 @@ public class TezLauncher extends Launche
pc.getProperties().setProperty(TezConfiguration.TEZ_AM_DAG_SCHEDULER_CLASS, DAGSchedulerNaturalOrderControlled.class.getName());
}
Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties(), true);
+ // Make sure MR counter does not exceed limit
+ if (conf.get(TezConfiguration.TEZ_COUNTERS_MAX) != null) {
+ conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY, Math.max(
+ conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTER_GROUPS_MAX_KEY, 0),
+ conf.getInt(TezConfiguration.TEZ_COUNTERS_MAX, 0)));
+ }
+ if (conf.get(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS) != null) {
+ conf.setInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, Math.max(
+ conf.getInt(org.apache.hadoop.mapreduce.MRJobConfig.COUNTERS_MAX_KEY, 0),
+ conf.getInt(TezConfiguration.TEZ_COUNTERS_MAX_GROUPS, 0)));
+ }
+ Limits.init(conf);
if (pc.defaultParallel == -1 && !conf.getBoolean(PigConfiguration.PIG_TEZ_AUTO_PARALLELISM, true)) {
pc.defaultParallel = 1;
}