You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by rk...@apache.org on 2014/12/20 00:44:41 UTC
hadoop git commit: MAPREDUCE-6199. AbstractCounters are not reset
completely on deserialization (adhoot via rkanter)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 8ee40a158 -> f9341c1e2
MAPREDUCE-6199. AbstractCounters are not reset completely on deserialization (adhoot via rkanter)
(cherry picked from commit 390a7c12f543b2c94a74f08d6d2a28410472043a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9341c1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9341c1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9341c1e
Branch: refs/heads/branch-2
Commit: f9341c1e2cffc4e3ac37cb7da4e535ca5c6d3913
Parents: 8ee40a1
Author: Robert Kanter <rk...@apache.org>
Authored: Fri Dec 19 15:43:48 2014 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Fri Dec 19 15:44:01 2014 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../mapreduce/counters/AbstractCounters.java | 4 ++
.../hadoop/mapreduce/counters/Limits.java | 7 ++++
.../apache/hadoop/mapreduce/TestCounters.java | 39 +++++++++++++++++++-
4 files changed, 52 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9341c1e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 2db4cbd..6721e56 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -58,6 +58,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6045. need close the DataInputStream after open it in
TestMapReduce.java (zxu via rkanter)
+ MAPREDUCE-6199. AbstractCounters are not reset completely on
+ deserialization (adhoot via rkanter)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9341c1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
index 401bbb2..dd81ebb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
@@ -307,6 +307,10 @@ public abstract class AbstractCounters<C extends Counter,
fgroups.put(group.getName(), group);
}
int numGroups = WritableUtils.readVInt(in);
+ if (!groups.isEmpty()) {
+ groups.clear();
+ limits.reset();
+ }
while (numGroups-- > 0) {
limits.checkGroups(groups.size() + 1);
G group = groupFactory.newGenericGroup(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9341c1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
index 3821694..9546c8d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
@@ -124,8 +124,15 @@ public class Limits {
return firstViolation;
}
+ // This allows initialization of global settings and not for an instance
public static synchronized void reset(Configuration conf) {
isInited = false;
init(conf);
}
+
+ // This allows resetting of an instance to allow reuse
+ public synchronized void reset() {
+ totalCounters = 0;
+ firstViolation = null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9341c1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
index 83d689c..0215568 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
@@ -17,8 +17,12 @@
*/
package org.apache.hadoop.mapreduce;
+import java.io.IOException;
import java.util.Random;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -70,7 +74,40 @@ public class TestCounters {
testMaxGroups(new Counters());
}
}
-
+
+ @Test public void testResetOnDeserialize() throws IOException {
+ // Allow only one counterGroup
+ Configuration conf = new Configuration();
+ conf.setInt(MRJobConfig.COUNTER_GROUPS_MAX_KEY, 1);
+ Limits.init(conf);
+
+ Counters countersWithOneGroup = new Counters();
+ countersWithOneGroup.findCounter("firstOf1Allowed", "First group");
+ boolean caughtExpectedException = false;
+ try {
+ countersWithOneGroup.findCounter("secondIsTooMany", "Second group");
+ }
+ catch (LimitExceededException _) {
+ caughtExpectedException = true;
+ }
+
+ assertTrue("Did not throw expected exception",
+ caughtExpectedException);
+
+ Counters countersWithZeroGroups = new Counters();
+ DataOutputBuffer out = new DataOutputBuffer();
+ countersWithZeroGroups.write(out);
+
+ DataInputBuffer in = new DataInputBuffer();
+ in.reset(out.getData(), out.getLength());
+
+ countersWithOneGroup.readFields(in);
+
+ // After reset one should be able to add a group
+ countersWithOneGroup.findCounter("firstGroupAfterReset", "After reset " +
+ "limit should be set back to zero");
+ }
+
@Test
public void testCountersIncrement() {
Counters fCounters = new Counters();