You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 06:55:21 UTC
svn commit: r1079203 - in /hadoop/mapreduce/branches/yahoo-merge/src:
java/org/apache/hadoop/mapreduce/counters/
test/mapred/org/apache/hadoop/mapreduce/
Author: omalley
Date: Tue Mar 8 05:55:21 2011
New Revision: 1079203
URL: http://svn.apache.org/viewvc?rev=1079203&view=rev
Log:
commit fe76e91e01b991d1b2401e9a08a697c9dc5f94ac
Author: Luke Lu <ll...@yahoo-inc.com>
Date: Tue Dec 7 03:19:56 2010 -0800
Fix counters limits
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java
hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1079203&r1=1079202&r2=1079203&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java Tue Mar 8 05:55:21 2011
@@ -78,6 +78,8 @@ public abstract class AbstractCounters<C
JobCounter.class.getName());
}
+ private final Limits limits = new Limits();
+
@InterfaceAudience.Private
public AbstractCounters(CounterGroupFactory<C, G> gf) {
groupFactory = gf;
@@ -97,7 +99,7 @@ public abstract class AbstractCounters<C
this.groupFactory = groupFactory;
for(G1 group: counters) {
String name = group.getName();
- G newGroup = groupFactory.newGroup(name, group.getDisplayName());
+ G newGroup = groupFactory.newGroup(name, group.getDisplayName(), limits);
(isFrameworkGroup(name) ? fgroups : groups).put(name, newGroup);
for(Counter counter: group) {
newGroup.addCounter(counter.getName(), counter.getDisplayName(),
@@ -129,7 +131,7 @@ public abstract class AbstractCounters<C
*/
@InterfaceAudience.Private
public G addGroup(String name, String displayName) {
- return addGroup(groupFactory.newGroup(name, displayName));
+ return addGroup(groupFactory.newGroup(name, displayName, limits));
}
/**
@@ -194,11 +196,11 @@ public abstract class AbstractCounters<C
boolean isFGroup = isFrameworkGroup(groupName);
G group = isFGroup ? fgroups.get(groupName) : groups.get(groupName);
if (group == null) {
- group = groupFactory.newGroup(filterGroupName(groupName));
+ group = groupFactory.newGroup(filterGroupName(groupName), limits);
if (isFGroup) {
fgroups.put(groupName, group);
} else {
- groupFactory.limits().checkGroups(groups.size() + 1);
+ limits.checkGroups(groups.size() + 1);
groups.put(groupName, group);
}
}
@@ -208,7 +210,7 @@ public abstract class AbstractCounters<C
private String filterGroupName(String oldName) {
String newName = legacyMap.get(oldName);
if (newName == null) {
- return groupFactory.limits().filterGroupName(oldName);
+ return limits.filterGroupName(oldName);
}
LOG.warn("Group "+ oldName +" is deprecated. Use "+ newName +" instead");
return newName;
@@ -284,9 +286,8 @@ public abstract class AbstractCounters<C
}
int numGroups = WritableUtils.readVInt(in);
while (numGroups-- > 0) {
- groupFactory.limits().checkGroups(groups.size() + 1);
- G group = groupFactory.newGenericGroup(Text.readString(in), null,
- groupFactory.limits());
+ limits.checkGroups(groups.size() + 1);
+ G group = groupFactory.newGenericGroup(Text.readString(in), null, limits);
group.readFields(in);
groups.put(group.getName(), group);
}
@@ -318,8 +319,9 @@ public abstract class AbstractCounters<C
for(G right : other) {
G left = groups.get(right.getName());
if (left == null) {
- groupFactory.limits().checkGroups(groups.size() + 1);
- left = groupFactory.newGroup(right.getName(), right.getDisplayName());
+ limits.checkGroups(groups.size() + 1);
+ left = groupFactory.newGroup(right.getName(), right.getDisplayName(),
+ limits);
groups.put(right.getName(), left);
}
left.incrAllCounters(right);
@@ -363,6 +365,6 @@ public abstract class AbstractCounters<C
@InterfaceAudience.Private
public Limits limits() {
- return groupFactory.limits();
+ return limits;
}
}
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java?rev=1079203&r1=1079202&r2=1079203&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java Tue Mar 8 05:55:21 2011
@@ -51,8 +51,6 @@ public abstract class CounterGroupFactor
private static final List<String> i2s = Lists.newArrayList();
private static final int VERSION = 1;
- private final Limits limits = new Limits();
-
private final Map<String, FrameworkGroupFactory<G>> fmap = Maps.newHashMap();
{
// Add builtin counter class here and the version when changed.
@@ -88,19 +86,22 @@ public abstract class CounterGroupFactor
/**
* Create a new counter group
* @param name of the group
+ * @param limits the counters limits policy object
* @return a new counter group
*/
- public G newGroup(String name) {
- return newGroup(name, ResourceBundles.getCounterGroupName(name, name));
+ public G newGroup(String name, Limits limits) {
+ return newGroup(name, ResourceBundles.getCounterGroupName(name, name),
+ limits);
}
/**
* Create a new counter group
* @param name of the group
* @param displayName of the group
+ * @param limits the counters limits policy object
* @return a new counter group
*/
- public G newGroup(String name, String displayName) {
+ public G newGroup(String name, String displayName, Limits limits) {
FrameworkGroupFactory<G> gf = fmap.get(name);
if (gf != null) return gf.newGroup(name);
if (name.equals(FileSystemCounter.class.getName())) {
@@ -184,9 +185,4 @@ public abstract class CounterGroupFactor
* @return a new file system counter group
*/
protected abstract G newFileSystemGroup();
-
- @InterfaceAudience.Private
- public Limits limits() {
- return limits;
- }
}
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java?rev=1079203&r1=1079202&r2=1079203&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java Tue Mar 8 05:55:21 2011
@@ -26,14 +26,14 @@ import static org.apache.hadoop.mapreduc
public class Limits {
static final Configuration conf = new Configuration();
- static final int GROUP_NAME_MAX = conf.getInt(COUNTER_GROUP_NAME_MAX_KEY,
- COUNTER_GROUP_NAME_MAX_DEFAULT);
- static final int COUNTER_NAME_MAX = conf.getInt(COUNTER_NAME_MAX_KEY,
- COUNTER_NAME_MAX_DEFAULT);
- static final int GROUPS_MAX = conf.getInt(COUNTER_GROUPS_MAX_KEY,
- COUNTER_GROUPS_MAX_DEFAULT);
- static final int COUNTERS_MAX = conf.getInt(COUNTERS_MAX_KEY,
- COUNTERS_MAX_DEFAULT);
+ public static final int GROUP_NAME_MAX =
+ conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, COUNTER_GROUP_NAME_MAX_DEFAULT);
+ public static final int COUNTER_NAME_MAX =
+ conf.getInt(COUNTER_NAME_MAX_KEY, COUNTER_NAME_MAX_DEFAULT);
+ public static final int GROUPS_MAX =
+ conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT);
+ public static final int COUNTERS_MAX =
+ conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT);
private int totalCounters;
private LimitExceededException firstViolation;
Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java?rev=1079203&r1=1079202&r2=1079203&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java Tue Mar 8 05:55:21 2011
@@ -22,11 +22,18 @@ import java.util.Random;
import org.junit.Test;
import static org.junit.Assert.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.counters.LimitExceededException;
+import org.apache.hadoop.mapreduce.counters.Limits;
+
/**
* TestCounters checks the sanity and recoverability of {@code Counters}
*/
public class TestCounters {
+ static final Log LOG = LogFactory.getLog(TestCounters.class);
+
/**
* Verify counter value works
*/
@@ -56,4 +63,69 @@ public class TestCounters {
}
}
+ @Test public void testLimits() {
+ for (int i = 0; i < 3; ++i) {
+ // make sure limits apply to separate containers
+ testMaxCounters(new Counters());
+ testMaxGroups(new Counters());
+ }
+ }
+
+ static final Enum<?> FRAMEWORK_COUNTER = TaskCounter.CPU_MILLISECONDS;
+ static final long FRAMEWORK_COUNTER_VALUE = 8;
+ static final String FS_SCHEME = "HDFS";
+ static final FileSystemCounter FS_COUNTER = FileSystemCounter.BYTES_READ;
+ static final long FS_COUNTER_VALUE = 10;
+
+ private void testMaxCounters(final Counters counters) {
+ LOG.info("counters max="+ Limits.COUNTERS_MAX);
+ for (int i = 0; i < Limits.COUNTERS_MAX; ++i) {
+ counters.findCounter("test", "test"+ i);
+ }
+ setExpected(counters);
+ shouldThrow(LimitExceededException.class, new Runnable() {
+ public void run() {
+ counters.findCounter("test", "bad");
+ }
+ });
+ checkExpected(counters);
+ }
+
+ private void testMaxGroups(final Counters counters) {
+ LOG.info("counter groups max="+ Limits.GROUPS_MAX);
+ for (int i = 0; i < Limits.GROUPS_MAX; ++i) {
+ // assuming COUNTERS_MAX > GROUPS_MAX
+ counters.findCounter("test"+ i, "test");
+ }
+ setExpected(counters);
+ shouldThrow(LimitExceededException.class, new Runnable() {
+ public void run() {
+ counters.findCounter("bad", "test");
+ }
+ });
+ checkExpected(counters);
+ }
+
+ private void setExpected(Counters counters) {
+ counters.findCounter(FRAMEWORK_COUNTER).setValue(FRAMEWORK_COUNTER_VALUE);
+ counters.findCounter(FS_SCHEME, FS_COUNTER).setValue(FS_COUNTER_VALUE);
+ }
+
+ private void checkExpected(Counters counters) {
+ assertEquals(FRAMEWORK_COUNTER_VALUE,
+ counters.findCounter(FRAMEWORK_COUNTER).getValue());
+ assertEquals(FS_COUNTER_VALUE,
+ counters.findCounter(FS_SCHEME, FS_COUNTER).getValue());
+ }
+
+ private void shouldThrow(Class<? extends Exception> ecls, Runnable runnable) {
+ try {
+ runnable.run();
+ } catch (Exception e) {
+ assertSame(ecls, e.getClass());
+ LOG.info("got expected: "+ e);
+ return;
+ }
+ assertTrue("Should've thrown "+ ecls.getSimpleName(), false);
+ }
}