You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sa...@apache.org on 2013/09/18 15:58:38 UTC
svn commit: r1524417 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/a...
Author: sandy
Date: Wed Sep 18 13:58:37 2013
New Revision: 1524417
URL: http://svn.apache.org/r1524417
Log:
MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again in Limits (Sandy Ryza)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1524417&r1=1524416&r2=1524417&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed Sep 18 13:58:37 2013
@@ -27,6 +27,9 @@ Release 2.3.0 - UNRELEASED
MAPREDUCE-5484. YarnChild unnecessarily loads job conf twice (Sandy Ryza)
+ MAPREDUCE-5487. In task processes, JobConf is unnecessarily loaded again
+ in Limits (Sandy Ryza)
+
BUG FIXES
MAPREDUCE-5316. job -list-attempt-ids command does not handle illegal
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java?rev=1524417&r1=1524416&r2=1524417&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java Wed Sep 18 13:58:37 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.counters.Limits;
import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
@@ -76,6 +77,8 @@ class YarnChild {
LOG.debug("Child starting");
final JobConf job = new JobConf();
+ // Initing with our JobConf allows us to avoid loading confs twice
+ Limits.init(job);
job.addResource(MRJobConfig.JOB_CONF_FILE);
UserGroupInformation.setConfiguration(job);
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java?rev=1524417&r1=1524416&r2=1524417&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Counters.java Wed Sep 18 13:58:37 2013
@@ -62,8 +62,8 @@ import com.google.common.collect.Iterato
public class Counters
extends AbstractCounters<Counters.Counter, Counters.Group> {
- public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
- public static int MAX_GROUP_LIMIT = Limits.GROUPS_MAX;
+ public static int MAX_COUNTER_LIMIT = Limits.getCountersMax();
+ public static int MAX_GROUP_LIMIT = Limits.getGroupsMax();
private static HashMap<String, String> depricatedCounterMap =
new HashMap<String, String>();
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java?rev=1524417&r1=1524416&r2=1524417&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java Wed Sep 18 13:58:37 2013
@@ -28,37 +28,80 @@ import static org.apache.hadoop.mapreduc
public class Limits {
static final Configuration conf = new JobConf();
- public static final int GROUP_NAME_MAX =
- conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, COUNTER_GROUP_NAME_MAX_DEFAULT);
- public static final int COUNTER_NAME_MAX =
- conf.getInt(COUNTER_NAME_MAX_KEY, COUNTER_NAME_MAX_DEFAULT);
- public static final int GROUPS_MAX =
- conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT);
- public static final int COUNTERS_MAX =
- conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT);
private int totalCounters;
private LimitExceededException firstViolation;
+ private static boolean isInited;
+
+ private static int GROUP_NAME_MAX;
+ private static int COUNTER_NAME_MAX;
+ private static int GROUPS_MAX;
+ private static int COUNTERS_MAX;
+
+ public synchronized static void init(Configuration conf) {
+ if (!isInited) {
+ if (conf == null) {
+ conf = new JobConf();
+ }
+ GROUP_NAME_MAX = conf.getInt(COUNTER_GROUP_NAME_MAX_KEY,
+ COUNTER_GROUP_NAME_MAX_DEFAULT);
+ COUNTER_NAME_MAX = conf.getInt(COUNTER_NAME_MAX_KEY,
+ COUNTER_NAME_MAX_DEFAULT);
+ GROUPS_MAX = conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT);
+ COUNTERS_MAX = conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT);
+ }
+ isInited = true;
+ }
+
+ public static int getGroupNameMax() {
+ if (!isInited) {
+ init(null);
+ }
+ return GROUP_NAME_MAX;
+ }
+
+ public static int getCounterNameMax() {
+ if (!isInited) {
+ init(null);
+ }
+ return COUNTER_NAME_MAX;
+ }
+
+ public static int getGroupsMax() {
+ if (!isInited) {
+ init(null);
+ }
+ return GROUPS_MAX;
+ }
+
+ public static int getCountersMax() {
+ if (!isInited) {
+ init(null);
+ }
+ return COUNTERS_MAX;
+ }
+
public static String filterName(String name, int maxLen) {
return name.length() > maxLen ? name.substring(0, maxLen - 1) : name;
}
public static String filterCounterName(String name) {
- return filterName(name, COUNTER_NAME_MAX);
+ return filterName(name, getCounterNameMax());
}
public static String filterGroupName(String name) {
- return filterName(name, GROUP_NAME_MAX);
+ return filterName(name, getGroupNameMax());
}
public synchronized void checkCounters(int size) {
if (firstViolation != null) {
throw new LimitExceededException(firstViolation);
}
- if (size > COUNTERS_MAX) {
+ int countersMax = getCountersMax();
+ if (size > countersMax) {
firstViolation = new LimitExceededException("Too many counters: "+ size +
- " max="+ COUNTERS_MAX);
+ " max="+ countersMax);
throw firstViolation;
}
}
@@ -72,9 +115,10 @@ public class Limits {
if (firstViolation != null) {
throw new LimitExceededException(firstViolation);
}
- if (size > GROUPS_MAX) {
+ int groupsMax = getGroupsMax();
+ if (size > groupsMax) {
firstViolation = new LimitExceededException("Too many counter groups: "+
- size +" max="+ GROUPS_MAX);
+ size +" max="+ groupsMax);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java?rev=1524417&r1=1524416&r2=1524417&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java Wed Sep 18 13:58:37 2013
@@ -101,8 +101,8 @@ public class TestCounters {
static final long FS_COUNTER_VALUE = 10;
private void testMaxCounters(final Counters counters) {
- LOG.info("counters max="+ Limits.COUNTERS_MAX);
- for (int i = 0; i < Limits.COUNTERS_MAX; ++i) {
+ LOG.info("counters max="+ Limits.getCountersMax());
+ for (int i = 0; i < Limits.getCountersMax(); ++i) {
counters.findCounter("test", "test"+ i);
}
setExpected(counters);
@@ -115,8 +115,8 @@ public class TestCounters {
}
private void testMaxGroups(final Counters counters) {
- LOG.info("counter groups max="+ Limits.GROUPS_MAX);
- for (int i = 0; i < Limits.GROUPS_MAX; ++i) {
+ LOG.info("counter groups max="+ Limits.getGroupsMax());
+ for (int i = 0; i < Limits.getGroupsMax(); ++i) {
// assuming COUNTERS_MAX > GROUPS_MAX
counters.findCounter("test"+ i, "test");
}