You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/12/19 01:48:06 UTC
svn commit: r1050745 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/
lucene/src/java/org/apache/lucene/index/ lucene/src/test/org/apache/lucene/
lucene/src/test/org/apache/lucene/sea...
Author: rmuir
Date: Sun Dec 19 00:48:06 2010
New Revision: 1050745
URL: http://svn.apache.org/viewvc?rev=1050745&view=rev
Log:
LUCENE-2819: backport test improvements for uncaught exceptions
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java?rev=1050745&r1=1050744&r2=1050745&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/test/org/apache/lucene/xmlparser/TestParser.java Sun Dec 19 00:48:06 2010
@@ -18,6 +18,8 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.LuceneTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -36,23 +38,17 @@ import org.apache.lucene.util.LuceneTest
*/
public class TestParser extends LuceneTestCase {
-
- CoreParser builder;
- static Directory dir;
- // TODO: change to CURRENT and rewrite test (this needs to set QueryParser.enablePositionIncrements, too, for work with CURRENT):
- Analyzer analyzer=new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_24);
- IndexReader reader;
- private IndexSearcher searcher;
-
- /*
- * @see TestCase#setUp()
- */
- @Override
- public void setUp() throws Exception {
- super.setUp();
-
- //initialize the parser
- builder=new CorePlusExtensionsParser("contents",analyzer);
+ private static CoreParser builder;
+ private static Directory dir;
+ private static IndexReader reader;
+ private static IndexSearcher searcher;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ // TODO: rewrite test (this needs to set QueryParser.enablePositionIncrements, too, for work with CURRENT):
+ Analyzer analyzer=new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_24);
+ //initialize the parser
+ builder=new CorePlusExtensionsParser("contents",analyzer);
BufferedReader d = new BufferedReader(new InputStreamReader(TestParser.class.getResourceAsStream("reuters21578.txt")));
dir=newDirectory();
@@ -82,13 +78,17 @@ public class TestParser extends LuceneTe
- @Override
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void afterClass() throws Exception {
reader.close();
searcher.close();
dir.close();
- super.tearDown();
+ reader = null;
+ searcher = null;
+ dir = null;
+ builder = null;
}
+
public void testSimpleXML() throws ParserException, IOException
{
Query q=parse("TermQuery.xml");
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1050745&r1=1050744&r2=1050745&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Sun Dec 19 00:48:06 2010
@@ -527,6 +527,9 @@ public class ConcurrentMergeScheduler ex
/** Used for testing */
private static List<ConcurrentMergeScheduler> allInstances;
+
+ /** @deprecated remove all this test mode code in lucene 3.2! */
+ @Deprecated
public static void setTestMode() {
allInstances = new ArrayList<ConcurrentMergeScheduler>();
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java?rev=1050745&r1=1050744&r2=1050745&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java Sun Dec 19 00:48:06 2010
@@ -105,6 +105,5 @@ public class TestMergeSchedulerExternal
assertTrue(mergeCalled);
assertTrue(excCalled);
dir.close();
- assertTrue(ConcurrentMergeScheduler.anyUnhandledExceptions());
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java?rev=1050745&r1=1050744&r2=1050745&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java Sun Dec 19 00:48:06 2010
@@ -277,6 +277,7 @@ public class TestMultiSearcher extends L
indexSearcher2.close();
ramDirectory1.close();
ramDirectory2.close();
+ searcher.close();
}
/* uncomment this when the highest score is always normalized to 1.0, even when it was < 1.0
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java?rev=1050745&r1=1050744&r2=1050745&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java Sun Dec 19 00:48:06 2010
@@ -18,16 +18,34 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.util._TestUtil;
/**
* Unit tests for the ParallelMultiSearcher
*/
public class TestParallelMultiSearcher extends TestMultiSearcher {
+ List<ExecutorService> pools = new ArrayList<ExecutorService>();
+
+ @Override
+ public void tearDown() throws Exception {
+ for (ExecutorService exec : pools)
+ exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
+ pools.clear();
+ super.tearDown();
+ }
@Override
protected MultiSearcher getMultiSearcherInstance(Searcher[] searchers)
throws IOException {
- return new ParallelMultiSearcher(searchers);
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+ pools.add(exec);
+ return new ParallelMultiSearcher(exec, searchers);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=1050745&r1=1050744&r2=1050745&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java Sun Dec 19 00:48:06 2010
@@ -25,6 +25,10 @@ import java.util.BitSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.document.Document;
@@ -43,6 +47,7 @@ import org.apache.lucene.store.LockObtai
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
/**
* Unit tests for sorting code.
@@ -556,7 +561,8 @@ public class TestSort extends LuceneTest
// Don't close the multiSearcher. it would close the full searcher too!
// Do the same for a ParallelMultiSearcher
- Searcher parallelSearcher=new ParallelMultiSearcher (new Searchable[] { full });
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+ Searcher parallelSearcher=new ParallelMultiSearcher (exec, full);
sort.setSort (new SortField ("int", SortField.INT),
new SortField ("string", SortField.STRING),
@@ -567,7 +573,8 @@ public class TestSort extends LuceneTest
new SortField ("string", SortField.STRING),
new SortField ("float", SortField.FLOAT, true) );
assertMatches (parallelSearcher, queryG, sort, "ZYXW");
- // Don't close the parallelSearcher. it would close the full searcher too!
+ parallelSearcher.close();
+ exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
}
// test sorts using a series of fields
@@ -635,8 +642,11 @@ public class TestSort extends LuceneTest
// test a variety of sorts using a parallel multisearcher
public void testParallelMultiSort() throws Exception {
- Searcher searcher = new ParallelMultiSearcher (new Searchable[] { searchX, searchY });
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+ Searcher searcher = new ParallelMultiSearcher (exec, searchX, searchY);
runMultiSorts(searcher, false);
+ searcher.close();
+ exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
}
// test that the relevancy scores are the same even if
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java?rev=1050745&r1=1050744&r2=1050745&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java Sun Dec 19 00:48:06 2010
@@ -223,6 +223,11 @@ public abstract class LuceneTestCase ext
@AfterClass
public static void afterClassLuceneTestCaseJ4() {
+ int rogueThreads = threadCleanup("test class");
+ if (rogueThreads > 0) {
+ // TODO: fail here once the leaks are fixed.
+ System.err.println("RESOURCE LEAK: test class left " + rogueThreads + " thread(s) running");
+ }
Locale.setDefault(savedLocale);
TimeZone.setDefault(savedTimeZone);
System.clearProperty("solr.solr.home");
@@ -306,13 +311,13 @@ public abstract class LuceneTestCase ext
savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
+ testsFailed = true;
uncaughtExceptions.add(new UncaughtExceptionEntry(t, e));
if (savedUncaughtExceptionHandler != null)
savedUncaughtExceptionHandler.uncaughtException(t, e);
}
});
- ConcurrentMergeScheduler.setTestMode();
savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount();
}
@@ -341,6 +346,18 @@ public abstract class LuceneTestCase ext
assertTrue("ensure your setUp() calls super.setUp()!!!", setup);
setup = false;
BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount);
+ if (!getClass().getName().startsWith("org.apache.solr")) {
+ int rogueThreads = threadCleanup("test method: '" + getName() + "'");
+ if (rogueThreads > 0) {
+ System.err.println("RESOURCE LEAK: test method: '" + getName()
+ + "' left " + rogueThreads + " thread(s) running");
+ // TODO: fail, but print seed for now.
+ if (!testsFailed && uncaughtExceptions.isEmpty()) {
+ reportAdditionalFailureInfo();
+ }
+ }
+ }
+ Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler);
try {
if (!uncaughtExceptions.isEmpty()) {
@@ -368,19 +385,61 @@ public abstract class LuceneTestCase ext
// isolated in distinct test methods
assertSaneFieldCaches(getTestLabel());
- if (ConcurrentMergeScheduler.anyUnhandledExceptions()) {
- // Clear the failure so that we don't just keep
- // failing subsequent test cases
- ConcurrentMergeScheduler.clearUnhandledExceptions();
- fail("ConcurrentMergeScheduler hit unhandled exceptions");
- }
} finally {
purgeFieldCache(FieldCache.DEFAULT);
}
-
- Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler);
}
+ private final static int THREAD_STOP_GRACE_MSEC = 1000;
+ // jvm-wide list of 'rogue threads' we found, so they only get reported once.
+ private final static IdentityHashMap<Thread,Boolean> rogueThreads = new IdentityHashMap<Thread,Boolean>();
+
+ /**
+ * Looks for leftover running threads, trying to kill them off,
+ * so they don't fail future tests.
+ * returns the number of rogue threads that it found.
+ */
+ private static int threadCleanup(String context) {
+ // educated guess
+ Thread[] stillRunning = new Thread[Thread.activeCount()+1];
+ int threadCount = 0;
+ int rogueCount = 0;
+
+ if ((threadCount = Thread.enumerate(stillRunning)) > 1) {
+ while (threadCount == stillRunning.length) {
+ // truncated response
+ stillRunning = new Thread[stillRunning.length*2];
+ threadCount = Thread.enumerate(stillRunning);
+ }
+
+ for (int i = 0; i < threadCount; i++) {
+ Thread t = stillRunning[i];
+
+ if (t.isAlive() &&
+ !rogueThreads.containsKey(t) &&
+ t != Thread.currentThread() &&
+ // TODO: TimeLimitingCollector starts a thread statically.... WTF?!
+ !t.getName().equals("TimeLimitedCollector timer thread")) {
+ System.err.println("WARNING: " + context + " left thread running: " + t);
+ rogueThreads.put(t, true);
+ rogueCount++;
+ // wait on the thread to die of natural causes
+ try {
+ t.join(THREAD_STOP_GRACE_MSEC);
+ } catch (InterruptedException e) { e.printStackTrace(); }
+ // try to stop the thread:
+ t.setUncaughtExceptionHandler(null);
+ Thread.setDefaultUncaughtExceptionHandler(null);
+ t.interrupt();
+ try {
+ t.join(THREAD_STOP_GRACE_MSEC);
+ } catch (InterruptedException e) { e.printStackTrace(); }
+ }
+ }
+ }
+ return rogueCount;
+ }
+
/**
* Asserts that FieldCacheSanityChecker does not detect any
* problems with FieldCache.DEFAULT.