You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2016/01/15 00:52:32 UTC
hadoop git commit: HADOOP-12706.
TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out
occasionally. Contributed by Sangjin Lee and Colin Patrick McCabe (cherry
picked from commit cdf88952599a43b1ef5adda792bfb195c7529fad)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 4976aac46 -> dbed88237
HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out occasionally. Contributed by Sangjin Lee and Colin Patrick McCabe
(cherry picked from commit cdf88952599a43b1ef5adda792bfb195c7529fad)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dbed8823
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dbed8823
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dbed8823
Branch: refs/heads/branch-2.7
Commit: dbed88237c6d0d40d924f8fd93e1a49e0ef79408
Parents: 4976aac
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Jan 14 23:52:10 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Jan 14 23:52:10 2016 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 9 +++++++++
.../apache/hadoop/fs/FCStatisticsBaseTest.java | 21 +++++++++++++++-----
2 files changed, 25 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dbed8823/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8402425..d659681 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -42,6 +42,9 @@ Release 2.7.3 - UNRELEASED
HADOOP-12107. long running apps may have a huge number of StatisticsData
instances under FileSystem (Sangjin Lee via Ming Ma)
+ HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
+ times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -863,6 +866,12 @@ Release 2.6.4 - UNRELEASED
HADOOP-11252. RPC client does not time out by default.
(Wilfred Spiegelenburg and Masatake Iwasaki via aajisaka)
+ HADOOP-12107. long running apps may have a huge number of StatisticsData
+ instances under FileSystem (Sangjin Lee via Ming Ma)
+
+ HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
+ times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dbed8823/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
index 3e33362..2e208d2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
@@ -32,6 +32,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
@@ -46,6 +48,8 @@ import com.google.common.util.concurrent.Uninterruptibles;
* </p>
*/
public abstract class FCStatisticsBaseTest {
+ private static final Log LOG = LogFactory.getLog(FCStatisticsBaseTest.class);
+
static protected int blockSize = 512;
static protected int numBlocks = 1;
@@ -110,7 +114,7 @@ public abstract class FCStatisticsBaseTest {
fc.delete(filePath, true);
}
- @Test(timeout=60000)
+ @Test(timeout=70000)
public void testStatisticsThreadLocalDataCleanUp() throws Exception {
final Statistics stats = new Statistics("test");
// create a small thread pool to test the statistics
@@ -137,17 +141,24 @@ public abstract class FCStatisticsBaseTest {
es.shutdownNow();
es.awaitTermination(1, TimeUnit.MINUTES);
es = null;
- System.gc();
+ System.gc(); // force GC to garbage collect threads
- // wait for up to 10 seconds
+ // wait for up to 60 seconds
GenericTestUtils.waitFor(new Supplier<Boolean>() {
@Override
public Boolean get() {
int size = stats.getAllThreadLocalDataSize();
allDataSize.set(size);
- return size == 0;
+ if (size == 0) {
+ return true;
+ }
+ LOG.warn("not all references have been cleaned up; still " +
+ allDataSize.get() + " references left");
+ LOG.warn("triggering another GC");
+ System.gc();
+ return false;
}
- }, 1000, 10*1000);
+ }, 500, 60*1000);
Assert.assertEquals(0, allDataSize.get());
Assert.assertEquals(size, stats.getReadOps());
}