You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/12/30 22:43:52 UTC
svn commit: r1225932 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/facet/src/test/org/apache/lucene/facet/search/
lucene/src/test-framework/java/org/apache/lucene/search/
lucene/src/test-framework/java/org/apache/lucene/store/ lucene/s...
Author: rmuir
Date: Fri Dec 30 21:43:51 2011
New Revision: 1225932
URL: http://svn.apache.org/viewvc?rev=1225932&view=rev
Log:
LUCENE-3667: partial backport
Added:
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java
- copied, changed from r1225920, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/build.xml
lucene/dev/branches/branch_3x/lucene/common-build.xml
lucene/dev/branches/branch_3x/lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestStressNRT.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermVectors.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/TestLockFactory.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
Modified: lucene/dev/branches/branch_3x/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/build.xml?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/build.xml (original)
+++ lucene/dev/branches/branch_3x/lucene/build.xml Fri Dec 30 21:43:51 2011
@@ -155,14 +155,10 @@
<target name="junit-backwards-parallel" unless="tests.sequential">
<parallel threadsPerProcessor="${tests.threadspercpu}">
- <backwards-test-macro threadNum="1" threadTotal="8"/>
- <backwards-test-macro threadNum="2" threadTotal="8"/>
- <backwards-test-macro threadNum="3" threadTotal="8"/>
- <backwards-test-macro threadNum="4" threadTotal="8"/>
- <backwards-test-macro threadNum="5" threadTotal="8"/>
- <backwards-test-macro threadNum="6" threadTotal="8"/>
- <backwards-test-macro threadNum="7" threadTotal="8"/>
- <backwards-test-macro threadNum="8" threadTotal="8"/>
+ <backwards-test-macro threadNum="1" threadTotal="4"/>
+ <backwards-test-macro threadNum="2" threadTotal="4"/>
+ <backwards-test-macro threadNum="3" threadTotal="4"/>
+ <backwards-test-macro threadNum="4" threadTotal="4"/>
</parallel>
</target>
Modified: lucene/dev/branches/branch_3x/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/common-build.xml?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/common-build.xml (original)
+++ lucene/dev/branches/branch_3x/lucene/common-build.xml Fri Dec 30 21:43:51 2011
@@ -635,14 +635,10 @@
<target name="junit-parallel" unless="tests.sequential">
<parallel threadsPerProcessor="${tests.threadspercpu}">
- <test-macro threadNum="1" threadTotal="8"/>
- <test-macro threadNum="2" threadTotal="8"/>
- <test-macro threadNum="3" threadTotal="8"/>
- <test-macro threadNum="4" threadTotal="8"/>
- <test-macro threadNum="5" threadTotal="8"/>
- <test-macro threadNum="6" threadTotal="8"/>
- <test-macro threadNum="7" threadTotal="8"/>
- <test-macro threadNum="8" threadTotal="8"/>
+ <test-macro threadNum="1" threadTotal="4"/>
+ <test-macro threadNum="2" threadTotal="4"/>
+ <test-macro threadNum="3" threadTotal="4"/>
+ <test-macro threadNum="4" threadTotal="4"/>
</parallel>
</target>
Modified: lucene/dev/branches/branch_3x/lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java Fri Dec 30 21:43:51 2011
@@ -132,4 +132,9 @@ public class TestTopKResultsHandlerRando
}
}
+ @Override
+ protected int numDocsToIndex() {
+ return TEST_NIGHTLY ? 20000 : 1000;
+ }
+
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java Fri Dec 30 21:43:51 2011
@@ -26,6 +26,7 @@ import junit.framework.Assert;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
public class CheckHits {
@@ -175,7 +176,7 @@ public class CheckHits {
Assert.assertEquals(query.toString(defaultFieldName), correct, actual);
- QueryUtils.check(random, query,searcher);
+ QueryUtils.check(random, query,searcher, LuceneTestCase.rarely(random));
}
/** Tests that a Hits has an expected order of documents */
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java Fri Dec 30 21:43:51 2011
@@ -106,7 +106,7 @@ public class QueryUtils {
public static void check(Random random, Query q1, Searcher s) {
check(random, q1, s, true);
}
- private static void check(Random random, Query q1, Searcher s, boolean wrap) {
+ public static void check(Random random, Query q1, Searcher s, boolean wrap) {
try {
check(q1);
if (s!=null) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java Fri Dec 30 21:43:51 2011
@@ -167,7 +167,10 @@ public class MockDirectoryWrapper extend
if (crashed)
throw new IOException("cannot sync after crash");
unSyncedFiles.removeAll(names);
- delegate.sync(names);
+ if (LuceneTestCase.rarely(randomState) || delegate instanceof NRTCachingDirectory) {
+ // don't wear out our hardware so much in tests.
+ delegate.sync(names);
+ }
}
@Override
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java Fri Dec 30 21:43:51 2011
@@ -600,7 +600,7 @@ public abstract class LuceneTestCase ext
}
}
- private final static int THREAD_STOP_GRACE_MSEC = 50;
+ private final static int THREAD_STOP_GRACE_MSEC = 10;
// jvm-wide list of 'rogue threads' we found, so they only get reported once.
private final static IdentityHashMap<Thread,Boolean> rogueThreads = new IdentityHashMap<Thread,Boolean>();
@@ -837,10 +837,10 @@ public abstract class LuceneTestCase ext
if (r.nextBoolean()) {
if (rarely(r)) {
// crazy value
- c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 7));
+ c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 15));
} else {
// reasonable value
- c.setMaxBufferedDocs(_TestUtil.nextInt(r, 8, 1000));
+ c.setMaxBufferedDocs(_TestUtil.nextInt(r, 16, 1000));
}
}
if (r.nextBoolean()) {
@@ -853,11 +853,19 @@ public abstract class LuceneTestCase ext
}
}
if (r.nextBoolean()) {
- c.setMaxThreadStates(_TestUtil.nextInt(r, 1, 20));
+ if (rarely(r)) {
+ // crazy value
+ c.setMaxThreadStates(_TestUtil.nextInt(r, 5, 20));
+ } else {
+ // reasonable value
+ c.setMaxThreadStates(_TestUtil.nextInt(r, 1, 4));
+ }
}
- if (r.nextBoolean()) {
+ if (rarely(r)) {
c.setMergePolicy(new MockRandomMergePolicy(r));
+ } else if (r.nextBoolean()) {
+ c.setMergePolicy(newTieredMergePolicy());
} else {
c.setMergePolicy(newLogMergePolicy());
}
@@ -880,9 +888,9 @@ public abstract class LuceneTestCase ext
logmp.setUseCompoundFile(r.nextBoolean());
logmp.setCalibrateSizeByDeletes(r.nextBoolean());
if (rarely(r)) {
- logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 4));
+ logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 9));
} else {
- logmp.setMergeFactor(_TestUtil.nextInt(r, 5, 50));
+ logmp.setMergeFactor(_TestUtil.nextInt(r, 10, 50));
}
return logmp;
}
@@ -890,16 +898,24 @@ public abstract class LuceneTestCase ext
public static TieredMergePolicy newTieredMergePolicy(Random r) {
TieredMergePolicy tmp = new TieredMergePolicy();
if (rarely(r)) {
- tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 4));
- tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 4));
+ tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 9));
+ tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 9));
+ } else {
+ tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 10, 50));
+ tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 10, 50));
+ }
+ if (rarely(r)) {
+ tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0);
} else {
- tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 5, 50));
- tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 5, 50));
+ tmp.setMaxMergedSegmentMB(r.nextDouble() * 100);
}
- tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0);
tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0);
tmp.setForceMergeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0);
- tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20));
+ if (rarely(r)) {
+ tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20));
+ } else {
+ tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 10, 50));
+ }
tmp.setUseCompoundFile(r.nextBoolean());
tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
tmp.setReclaimDeletesWeight(r.nextDouble()*4);
@@ -1186,7 +1202,7 @@ public abstract class LuceneTestCase ext
* with one that returns null for getSequentialSubReaders.
*/
public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException {
- if (random.nextBoolean()) {
+ if (usually()) {
if (maybeWrap && rarely()) {
r = new SlowMultiReaderWrapper(r);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestCompoundFile.java Fri Dec 30 21:43:51 2011
@@ -655,7 +655,7 @@ public class TestCompoundFile extends Lu
public void testManySubFiles() throws IOException {
final Directory d = newFSDirectory(_TestUtil.getTempDir("CFSManySubFiles"));
- final int FILE_COUNT = 10000;
+ final int FILE_COUNT = atLeast(500);
for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
IndexOutput out = d.createOutput("file." + fileIdx);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Dec 30 21:43:51 2011
@@ -59,9 +59,11 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
+import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.SimpleFSLockFactory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
@@ -597,7 +599,8 @@ public class TestIndexWriter extends Luc
public void testDiverseDocs() throws IOException {
MockDirectoryWrapper dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setRAMBufferSizeMB(0.5));
- for(int i=0;i<3;i++) {
+ int n = atLeast(1);
+ for(int i=0;i<n;i++) {
// First, docs where every term is unique (heavy on
// Posting instances)
for(int j=0;j<100;j++) {
@@ -636,7 +639,7 @@ public class TestIndexWriter extends Luc
IndexReader reader = IndexReader.open(dir, false);
IndexSearcher searcher = new IndexSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(new Term("field", "aaa")), null, 1000).scoreDocs;
- assertEquals(300, hits.length);
+ assertEquals(n*100, hits.length);
searcher.close();
reader.close();
@@ -925,104 +928,6 @@ public class TestIndexWriter extends Luc
dir.close();
}
- public void testNoWaitClose() throws Throwable {
- Directory directory = newDirectory();
-
- final Document doc = new Document();
- Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
- doc.add(idField);
-
- for(int pass=0;pass<2;pass++) {
- if (VERBOSE) {
- System.out.println("TEST: pass=" + pass);
- }
-
- IndexWriterConfig conf = newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)
- .setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
- if (pass == 2) {
- conf.setMergeScheduler(new SerialMergeScheduler());
- }
- IndexWriter writer = new IndexWriter(directory, conf);
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(100);
- writer.setInfoStream(VERBOSE ? System.out : null);
-
- // have to use compound file to prevent running out of
- // descripters when newDirectory returns a file-system
- // backed directory:
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true);
-
- for(int iter=0;iter<10;iter++) {
- if (VERBOSE) {
- System.out.println("TEST: iter=" + iter);
- }
- for(int j=0;j<199;j++) {
- idField.setValue(Integer.toString(iter*201+j));
- writer.addDocument(doc);
- }
-
- int delID = iter*199;
- for(int j=0;j<20;j++) {
- writer.deleteDocuments(new Term("id", Integer.toString(delID)));
- delID += 5;
- }
-
- // Force a bunch of merge threads to kick off so we
- // stress out aborting them on close:
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);
-
- final IndexWriter finalWriter = writer;
- final ArrayList<Throwable> failure = new ArrayList<Throwable>();
- Thread t1 = new Thread() {
- @Override
- public void run() {
- boolean done = false;
- while(!done) {
- for(int i=0;i<100;i++) {
- try {
- finalWriter.addDocument(doc);
- } catch (AlreadyClosedException e) {
- done = true;
- break;
- } catch (NullPointerException e) {
- done = true;
- break;
- } catch (Throwable e) {
- e.printStackTrace(System.out);
- failure.add(e);
- done = true;
- break;
- }
- }
- Thread.yield();
- }
-
- }
- };
-
- if (failure.size() > 0) {
- throw failure.get(0);
- }
-
- t1.start();
-
- writer.close(false);
- t1.join();
-
- // Make sure reader can read
- IndexReader reader = IndexReader.open(directory, true);
- reader.close();
-
- // Reopen
- writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
- writer.setInfoStream(VERBOSE ? System.out : null);
- }
- writer.close();
- }
-
- directory.close();
- }
-
// LUCENE-1084: test unlimited field length
public void testUnlimitedMaxFieldLength() throws IOException {
Directory dir = newDirectory();
@@ -1782,97 +1687,6 @@ public class TestIndexWriter extends Luc
dir.close();
}
- public void testRandomStoredFields() throws IOException {
- Directory dir = newDirectory();
- Random rand = random;
- RandomIndexWriter w = new RandomIndexWriter(rand, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(_TestUtil.nextInt(rand, 5, 20)));
- //w.w.setInfoStream(System.out);
- //w.w.setUseCompoundFile(false);
- if (VERBOSE) {
- w.w.setInfoStream(System.out);
- }
- final int docCount = atLeast(200);
- final int fieldCount = _TestUtil.nextInt(rand, 1, 5);
-
- final List<Integer> fieldIDs = new ArrayList<Integer>();
-
- Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
-
- for(int i=0;i<fieldCount;i++) {
- fieldIDs.add(i);
- }
-
- final Map<String,Document> docs = new HashMap<String,Document>();
-
- if (VERBOSE) {
- System.out.println("TEST: build index docCount=" + docCount);
- }
-
- for(int i=0;i<docCount;i++) {
- Document doc = new Document();
- doc.add(idField);
- final String id = ""+i;
- idField.setValue(id);
- docs.put(id, doc);
-
- for(int field: fieldIDs) {
- final String s;
- if (rand.nextInt(4) != 3) {
- s = _TestUtil.randomUnicodeString(rand, 1000);
- doc.add(newField("f"+field, s, Field.Store.YES, Field.Index.NO));
- } else {
- s = null;
- }
- }
- w.addDocument(doc);
- if (rand.nextInt(50) == 17) {
- // mixup binding of field name -> Number every so often
- Collections.shuffle(fieldIDs);
- }
- if (rand.nextInt(5) == 3 && i > 0) {
- final String delID = ""+rand.nextInt(i);
- if (VERBOSE) {
- System.out.println("TEST: delete doc " + delID);
- }
- w.deleteDocuments(new Term("id", delID));
- docs.remove(delID);
- }
- }
-
- if (VERBOSE) {
- System.out.println("TEST: " + docs.size() + " docs in index; now load fields");
- }
- if (docs.size() > 0) {
- String[] idsList = docs.keySet().toArray(new String[docs.size()]);
-
- for(int x=0;x<2;x++) {
- IndexReader r = w.getReader();
- IndexSearcher s = newSearcher(r);
-
- if (VERBOSE) {
- System.out.println("TEST: cycle x=" + x + " r=" + r);
- }
-
- int num = atLeast(1000);
- for(int iter=0;iter<num;iter++) {
- String testID = idsList[rand.nextInt(idsList.length)];
- TopDocs hits = s.search(new TermQuery(new Term("id", testID)), 1);
- assertEquals(1, hits.totalHits);
- Document doc = r.document(hits.scoreDocs[0].doc);
- Document docExp = docs.get(testID);
- for(int i=0;i<fieldCount;i++) {
- assertEquals("doc " + testID + ", field f" + fieldCount + " is wrong", docExp.get("f"+i), doc.get("f"+i));
- }
- }
- s.close();
- r.close();
- w.forceMerge(1);
- }
- }
- w.close();
- dir.close();
- }
-
public void testNoUnwantedTVFiles() throws Exception {
Directory dir = newDirectory();
@@ -2018,4 +1832,22 @@ public class TestIndexWriter extends Luc
assert(version3 > version2);
d.close();
}
+
+ public void testWhetherDeleteAllDeletesWriteLock() throws Exception {
+ Directory d = newFSDirectory(_TestUtil.getTempDir("TestIndexWriter.testWhetherDeleteAllDeletesWriteLock"));
+ // Must use SimpleFSLockFactory... NativeFSLockFactory
+ // somehow "knows" a lock is held against write.lock
+ // even if you remove that file:
+ d.setLockFactory(new SimpleFSLockFactory());
+ RandomIndexWriter w1 = new RandomIndexWriter(random, d);
+ w1.deleteAll();
+ try {
+ new RandomIndexWriter(random, d, newIndexWriterConfig(TEST_VERSION_CURRENT, null).setWriteLockTimeout(100));
+ fail("should not be able to create another writer");
+ } catch (LockObtainFailedException lofe) {
+ // expected
+ }
+ w1.close();
+ d.close();
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Fri Dec 30 21:43:51 2011
@@ -945,7 +945,7 @@ public class TestIndexWriterDelete exten
// note: tiny rambuffer used, as with a 1MB buffer the test is too slow (flush @ 128,999)
IndexWriter w = new IndexWriter(dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setRAMBufferSizeMB(0.2f).setMaxBufferedDocs(1000).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false));
+ .setRAMBufferSizeMB(0.1f).setMaxBufferedDocs(1000).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false));
w.setInfoStream(VERBOSE ? System.out : null);
int count = 0;
while(true) {
@@ -1025,6 +1025,7 @@ public class TestIndexWriterDelete exten
// Make sure buffered (pushed) deletes don't use up so
// much RAM that it forces long tail of tiny segments:
+ @Nightly
public void testApplyDeletesOnFlush() throws Exception {
Directory dir = newDirectory();
// Cannot use RandomIndexWriter because we don't want to
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java Fri Dec 30 21:43:51 2011
@@ -15,6 +15,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -26,6 +27,7 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.util.LuceneTestCase;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Random;
@@ -299,4 +301,102 @@ public class TestIndexWriterMerging exte
iw.close();
dir.close();
}
+
+ public void testNoWaitClose() throws Throwable {
+ Directory directory = newDirectory();
+
+ final Document doc = new Document();
+ Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
+ doc.add(idField);
+
+ for(int pass=0;pass<2;pass++) {
+ if (VERBOSE) {
+ System.out.println("TEST: pass=" + pass);
+ }
+
+ IndexWriterConfig conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)
+ .setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
+ if (pass == 2) {
+ conf.setMergeScheduler(new SerialMergeScheduler());
+ }
+ IndexWriter writer = new IndexWriter(directory, conf);
+ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(100);
+ writer.setInfoStream(VERBOSE ? System.out : null);
+
+ // have to use compound file to prevent running out of
+ // descripters when newDirectory returns a file-system
+ // backed directory:
+ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true);
+
+ for(int iter=0;iter<10;iter++) {
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + iter);
+ }
+ for(int j=0;j<199;j++) {
+ idField.setValue(Integer.toString(iter*201+j));
+ writer.addDocument(doc);
+ }
+
+ int delID = iter*199;
+ for(int j=0;j<20;j++) {
+ writer.deleteDocuments(new Term("id", Integer.toString(delID)));
+ delID += 5;
+ }
+
+ // Force a bunch of merge threads to kick off so we
+ // stress out aborting them on close:
+ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);
+
+ final IndexWriter finalWriter = writer;
+ final ArrayList<Throwable> failure = new ArrayList<Throwable>();
+ Thread t1 = new Thread() {
+ @Override
+ public void run() {
+ boolean done = false;
+ while(!done) {
+ for(int i=0;i<100;i++) {
+ try {
+ finalWriter.addDocument(doc);
+ } catch (AlreadyClosedException e) {
+ done = true;
+ break;
+ } catch (NullPointerException e) {
+ done = true;
+ break;
+ } catch (Throwable e) {
+ e.printStackTrace(System.out);
+ failure.add(e);
+ done = true;
+ break;
+ }
+ }
+ Thread.yield();
+ }
+
+ }
+ };
+
+ if (failure.size() > 0) {
+ throw failure.get(0);
+ }
+
+ t1.start();
+
+ writer.close(false);
+ t1.join();
+
+ // Make sure reader can read
+ IndexReader reader = IndexReader.open(directory, true);
+ reader.close();
+
+ // Reopen
+ writer = new IndexWriter(directory, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
+ writer.setInfoStream(VERBOSE ? System.out : null);
+ }
+ writer.close();
+ }
+
+ directory.close();
+ }
}
Copied: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java (from r1225920, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java?p2=lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java&p1=lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java&r1=1225920&r2=1225932&rev=1225932&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestRandomStoredFields.java Fri Dec 30 21:43:51 2011
@@ -28,8 +28,6 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
@@ -43,43 +41,40 @@ public class TestRandomStoredFields exte
Directory dir = newDirectory();
Random rand = random;
RandomIndexWriter w = new RandomIndexWriter(rand, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(_TestUtil.nextInt(rand, 5, 20)));
+ //w.w.setInfoStream(System.out);
//w.w.setUseCompoundFile(false);
+ if (VERBOSE) {
+ w.w.setInfoStream(System.out);
+ }
final int docCount = atLeast(200);
final int fieldCount = _TestUtil.nextInt(rand, 1, 5);
-
+
final List<Integer> fieldIDs = new ArrayList<Integer>();
- FieldType customType = new FieldType(TextField.TYPE_STORED);
- customType.setTokenized(false);
- Field idField = newField("id", "", customType);
+ Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
for(int i=0;i<fieldCount;i++) {
fieldIDs.add(i);
}
final Map<String,Document> docs = new HashMap<String,Document>();
-
+
if (VERBOSE) {
System.out.println("TEST: build index docCount=" + docCount);
}
- FieldType customType2 = new FieldType();
- customType2.setStored(true);
for(int i=0;i<docCount;i++) {
Document doc = new Document();
doc.add(idField);
final String id = ""+i;
idField.setValue(id);
docs.put(id, doc);
- if (VERBOSE) {
- System.out.println("TEST: add doc id=" + id);
- }
for(int field: fieldIDs) {
final String s;
if (rand.nextInt(4) != 3) {
s = _TestUtil.randomUnicodeString(rand, 1000);
- doc.add(newField("f"+field, s, customType2));
+ doc.add(newField("f"+field, s, Field.Store.YES, Field.Index.NO));
} else {
s = null;
}
@@ -92,7 +87,7 @@ public class TestRandomStoredFields exte
if (rand.nextInt(5) == 3 && i > 0) {
final String delID = ""+rand.nextInt(i);
if (VERBOSE) {
- System.out.println("TEST: delete doc id=" + delID);
+ System.out.println("TEST: delete doc " + delID);
}
w.deleteDocuments(new Term("id", delID));
docs.remove(delID);
@@ -116,9 +111,6 @@ public class TestRandomStoredFields exte
int num = atLeast(1000);
for(int iter=0;iter<num;iter++) {
String testID = idsList[rand.nextInt(idsList.length)];
- if (VERBOSE) {
- System.out.println("TEST: test id=" + testID);
- }
TopDocs hits = s.search(new TermQuery(new Term("id", testID)), 1);
assertEquals(1, hits.totalHits);
Document doc = r.document(hits.scoreDocs[0].doc);
@@ -127,6 +119,7 @@ public class TestRandomStoredFields exte
assertEquals("doc " + testID + ", field f" + fieldCount + " is wrong", docExp.get("f"+i), doc.get("f"+i));
}
}
+ s.close();
r.close();
w.forceMerge(1);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestStressNRT.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestStressNRT.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestStressNRT.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestStressNRT.java Fri Dec 30 21:43:51 2011
@@ -77,7 +77,7 @@ public class TestStressNRT extends Lucen
// query variables
- final AtomicLong operations = new AtomicLong(atLeast(50000)); // number of query operations to perform in total
+ final AtomicLong operations = new AtomicLong(atLeast(10000)); // number of query operations to perform in total
final int nReadThreads = _TestUtil.nextInt(random, 1, TEST_NIGHTLY ? 10 : 5);
initModel(ndocs);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSort.java Fri Dec 30 21:43:51 2011
@@ -73,7 +73,7 @@ public class TestSort extends LuceneTest
@BeforeClass
public static void beforeClass() throws Exception {
- NUM_STRINGS = atLeast(6000);
+ NUM_STRINGS = atLeast(500);
}
// document data:
// the tracer field is used to determine which document was hit
@@ -154,8 +154,7 @@ public class TestSort extends LuceneTest
dirs.add(indexStore);
IndexWriter writer = new IndexWriter(
indexStore,
- new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDocs(4).
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setMergePolicy(newLogMergePolicy(97))
);
for (int i=0; i<NUM_STRINGS; i++) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermVectors.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermVectors.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermVectors.java Fri Dec 30 21:43:51 2011
@@ -30,6 +30,8 @@ import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.English;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import java.io.IOException;
import java.util.HashMap;
@@ -37,13 +39,12 @@ import java.util.Map;
import java.util.SortedSet;
public class TestTermVectors extends LuceneTestCase {
- private IndexSearcher searcher;
- private IndexReader reader;
- private Directory directory;
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
+ private static IndexSearcher searcher;
+ private static IndexReader reader;
+ private static Directory directory;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.SIMPLE, true)).setMergePolicy(newLogMergePolicy()));
//writer.setUseCompoundFile(true);
@@ -74,12 +75,14 @@ public class TestTermVectors extends Luc
searcher = newSearcher(reader);
}
- @Override
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void afterClass() throws Exception {
searcher.close();
reader.close();
directory.close();
- super.tearDown();
+ reader = null;
+ directory = null;
+ searcher = null;
}
public void test() {
@@ -383,7 +386,7 @@ public class TestTermVectors extends Luc
}
IndexReader reader = writer.getReader();
writer.close();
- searcher = newSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
Query query = new TermQuery(new Term("field", "hundred"));
ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
@@ -419,7 +422,7 @@ public class TestTermVectors extends Luc
IndexReader reader = writer.getReader();
writer.close();
- searcher = newSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
Query query = new TermQuery(new Term("field", "one"));
ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java Fri Dec 30 21:43:51 2011
@@ -294,11 +294,11 @@ public class TestCustomScoreQuery extend
assertEquals("queries should have same #hits",h1.size(),h4CustomAdd.size());
assertEquals("queries should have same #hits",h1.size(),h5CustomMulAdd.size());
- QueryUtils.check(random, q1,s);
- QueryUtils.check(random, q2,s);
- QueryUtils.check(random, q3,s);
- QueryUtils.check(random, q4,s);
- QueryUtils.check(random, q5,s);
+ QueryUtils.check(random, q1, s, rarely());
+ QueryUtils.check(random, q2, s, rarely());
+ QueryUtils.check(random, q3, s, rarely());
+ QueryUtils.check(random, q4, s, rarely());
+ QueryUtils.check(random, q5, s, rarely());
// verify scores ratios
for (final Integer doc : h1.keySet()) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java Fri Dec 30 21:43:51 2011
@@ -116,7 +116,7 @@ public class TestBasics extends LuceneTe
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, simplePayloadAnalyzer)
- .setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)).setMergePolicy(newLogMergePolicy()));
+ .setMaxBufferedDocs(_TestUtil.nextInt(random, 100, 1000)).setMergePolicy(newLogMergePolicy()));
//writer.infoStream = System.out;
for (int i = 0; i < 2000; i++) {
Document doc = new Document();
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/TestLockFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/TestLockFactory.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/TestLockFactory.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/TestLockFactory.java Fri Dec 30 21:43:51 2011
@@ -132,6 +132,7 @@ public class TestLockFactory extends Luc
// Verify: do stress test, by opening IndexReaders and
// IndexWriters over & over in 2 threads and making sure
// no unexpected exceptions are raised:
+ @Nightly
public void testStressLocks() throws Exception {
_testStressLocks(null, _TestUtil.getTempDir("index.TestLockFactory6"));
}
@@ -140,6 +141,7 @@ public class TestLockFactory extends Luc
// IndexWriters over & over in 2 threads and making sure
// no unexpected exceptions are raised, but use
// NativeFSLockFactory:
+ @Nightly
public void testStressLocksNativeFSLockFactory() throws Exception {
File dir = _TestUtil.getTempDir("index.TestLockFactory7");
_testStressLocks(new NativeFSLockFactory(dir), dir);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1225932&r1=1225931&r2=1225932&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java Fri Dec 30 21:43:51 2011
@@ -1443,7 +1443,7 @@ public class TestFSTs extends LuceneTest
Field idField = newField("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
doc.add(idField);
- final int NUM_IDS = (int) (1000*RANDOM_MULTIPLIER*(1.0+random.nextDouble()));
+ final int NUM_IDS = atLeast(200);
//final int NUM_IDS = (int) (377 * (1.0+random.nextDouble()));
if (VERBOSE) {
System.out.println("TEST: NUM_IDS=" + NUM_IDS);