You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2016/01/15 00:48:07 UTC
[1/2] hadoop git commit: Update CHANGES.txt for commit of
HADOOP-12107 to branch-2.7 and branch-2.6 (cherry picked from commit
651c23e8ef8aeafd999249ce57b31e689bd2ece6)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 aadcaf4f3 -> 81cda7608
Update CHANGES.txt for commit of HADOOP-12107 to branch-2.7 and branch-2.6
(cherry picked from commit 651c23e8ef8aeafd999249ce57b31e689bd2ece6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86456b01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86456b01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86456b01
Branch: refs/heads/branch-2
Commit: 86456b01eb112091dcbec0ca2d2c091a620ce055
Parents: aadcaf4
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Jan 14 23:41:14 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Jan 14 23:42:58 2016 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86456b01/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d425634..f18d85c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1022,6 +1022,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-12588. Fix intermittent test failure of TestGangliaMetrics.
(Masatake Iwasaki via aajisaka)
+ HADOOP-12107. long running apps may have a huge number of StatisticsData
+ instances under FileSystem (Sangjin Lee via Ming Ma)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1843,6 +1846,9 @@ Release 2.6.4 - UNRELEASED
HADOOP-11252. RPC client does not time out by default.
(Wilfred Spiegelenburg and Masatake Iwasaki via aajisaka)
+ HADOOP-12107. long running apps may have a huge number of StatisticsData
+ instances under FileSystem (Sangjin Lee via Ming Ma)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
[2/2] hadoop git commit: HADOOP-12706.
TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out
occasionally. Contributed by Sangjin Lee and Colin Patrick McCabe (cherry
picked from commit cdf88952599a43b1ef5adda792bfb195c7529fad)
Posted by jl...@apache.org.
HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out occasionally. Contributed by Sangjin Lee and Colin Patrick McCabe
(cherry picked from commit cdf88952599a43b1ef5adda792bfb195c7529fad)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81cda760
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81cda760
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81cda760
Branch: refs/heads/branch-2
Commit: 81cda7608b7c30e485fb2703d7429d46da85c182
Parents: 86456b0
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Jan 14 23:45:03 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Jan 14 23:47:29 2016 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 ++++++
.../apache/hadoop/fs/FCStatisticsBaseTest.java | 21 +++++++++++++++-----
2 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81cda760/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f18d85c..d5c8788 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1025,6 +1025,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-12107. long running apps may have a huge number of StatisticsData
instances under FileSystem (Sangjin Lee via Ming Ma)
+ HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
+ times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1849,6 +1852,9 @@ Release 2.6.4 - UNRELEASED
HADOOP-12107. long running apps may have a huge number of StatisticsData
instances under FileSystem (Sangjin Lee via Ming Ma)
+ HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
+ times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81cda760/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
index 3e33362..2e208d2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
@@ -32,6 +32,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
@@ -46,6 +48,8 @@ import com.google.common.util.concurrent.Uninterruptibles;
* </p>
*/
public abstract class FCStatisticsBaseTest {
+ private static final Log LOG = LogFactory.getLog(FCStatisticsBaseTest.class);
+
static protected int blockSize = 512;
static protected int numBlocks = 1;
@@ -110,7 +114,7 @@ public abstract class FCStatisticsBaseTest {
fc.delete(filePath, true);
}
- @Test(timeout=60000)
+ @Test(timeout=70000)
public void testStatisticsThreadLocalDataCleanUp() throws Exception {
final Statistics stats = new Statistics("test");
// create a small thread pool to test the statistics
@@ -137,17 +141,24 @@ public abstract class FCStatisticsBaseTest {
es.shutdownNow();
es.awaitTermination(1, TimeUnit.MINUTES);
es = null;
- System.gc();
+ System.gc(); // force GC to garbage collect threads
- // wait for up to 10 seconds
+ // wait for up to 60 seconds
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
int size = stats.getAllThreadLocalDataSize();
allDataSize.set(size);
- return size == 0;
+ if (size == 0) {
+ return true;
+ }
+ LOG.warn("not all references have been cleaned up; still " +
+ allDataSize.get() + " references left");
+ LOG.warn("triggering another GC");
+ System.gc();
+ return false;
}
- }, 1000, 10*1000);
+ }, 500, 60*1000);
Assert.assertEquals(0, allDataSize.get());
Assert.assertEquals(size, stats.getReadOps());
}