You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/03/12 23:52:55 UTC
git commit: TEZ-920. Increase defaults for number of counters,
ensure AM uses this. (sseth)
Repository: incubator-tez
Updated Branches:
refs/heads/master 3d0edf44e -> 0d65fabbc
TEZ-920. Increase defaults for number of counters, ensure AM uses this.
(sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/0d65fabb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/0d65fabb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/0d65fabb
Branch: refs/heads/master
Commit: 0d65fabbc84ed89bb1bfcccba0de2e31e813e92d
Parents: 3d0edf4
Author: Siddharth Seth <ss...@apache.org>
Authored: Wed Mar 12 15:52:30 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Wed Mar 12 15:52:30 2014 -0700
----------------------------------------------------------------------
tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java | 4 ++--
.../src/main/java/org/apache/tez/common/counters/Limits.java | 7 +++++++
.../src/main/java/org/apache/tez/dag/app/DAGAppMaster.java | 2 ++
3 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0d65fabb/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
index b34eaf2..69edb8b 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -145,7 +145,7 @@ public class TezJobConfig {
*
*/
public static final String COUNTERS_MAX_KEY = "tez.runtime.job.counters.max";
- public static final int COUNTERS_MAX_DEFAULT = 120;
+ public static final int COUNTERS_MAX_DEFAULT = 1200;
/**
*
@@ -163,7 +163,7 @@ public class TezJobConfig {
*
*/
public static final String COUNTER_GROUPS_MAX_KEY = "tez.runtime.job.counters.groups.max";
- public static final int COUNTER_GROUPS_MAX_DEFAULT = 50;
+ public static final int COUNTER_GROUPS_MAX_DEFAULT = 500;
/**
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0d65fabb/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
index aacce87..070c4c2 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
@@ -18,12 +18,16 @@
package org.apache.tez.common.counters;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.tez.common.TezJobConfig;
@InterfaceAudience.Private
public class Limits {
+
+ private static final Log LOG = LogFactory.getLog(Limits.class);
private static Configuration conf = null;
private static int GROUP_NAME_MAX;
@@ -52,6 +56,9 @@ public class Limits {
conf.getInt(TezJobConfig.COUNTERS_MAX_KEY, TezJobConfig.
COUNTERS_MAX_DEFAULT);
initialized = true;
+ LOG.info("Counter limits initialized with parameters: " + " GROUP_NAME_MAX=" + GROUP_NAME_MAX
+ + ", MAX_GROUPS=" + GROUPS_MAX + ", COUNTER_NAME_MAX=" + COUNTER_NAME_MAX
+ + ", MAX_COUNTERS=" + COUNTERS_MAX);
}
private int totalCounters;
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/0d65fabb/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index ae2bc64..06af9ca 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.tez.client.PreWarmContext;
import org.apache.tez.client.TezSessionStatus;
import org.apache.tez.common.TezUtils;
+import org.apache.tez.common.counters.Limits;
import org.apache.tez.common.impl.LogUtils;
import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.common.security.TokenCache;
@@ -1599,6 +1600,7 @@ public class DAGAppMaster extends AbstractService {
ShutdownHookManager.get().addShutdownHook(
new DAGAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
+ Limits.setConfiguration(conf);
initAndStartAppMaster(appMaster, conf,
jobUserName);