You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/10/07 09:59:16 UTC
svn commit: r1005356 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/index/DocumentsWriter.java
test/org/apache/lucene/index/Test2BTerms.java
test/org/apache/lucene/index/TestIndexWriter.java
test/org/apache/lucene/util/_TestUtil.java
Author: mikemccand
Date: Thu Oct 7 07:59:16 2010
New Revision: 1005356
URL: http://svn.apache.org/viewvc?rev=1005356&view=rev
Log:
LUCENE-2682: improve Test2BTerms to not generate pathological (sequential) terms for the BytesRefHash; they are generated fully randomly now
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1005356&r1=1005355&r2=1005356&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Thu Oct 7 07:59:16 2010
@@ -737,10 +737,12 @@ final class DocumentsWriter {
* been acquired. */
synchronized DocumentsWriterThreadState getThreadState(Document doc, Term delTerm) throws IOException {
+ final Thread currentThread = Thread.currentThread();
+
// First, find a thread state. If this thread already
// has affinity to a specific ThreadState, use that one
// again.
- DocumentsWriterThreadState state = threadBindings.get(Thread.currentThread());
+ DocumentsWriterThreadState state = threadBindings.get(currentThread);
if (state == null) {
// First time this thread has called us since last
@@ -762,7 +764,7 @@ final class DocumentsWriter {
state = newArray[threadStates.length] = new DocumentsWriterThreadState(this);
threadStates = newArray;
}
- threadBindings.put(Thread.currentThread(), state);
+ threadBindings.put(currentThread, state);
}
// Next, wait until my thread state is idle (in case
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1005356&r1=1005355&r2=1005356&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java Thu Oct 7 07:59:16 2010
@@ -34,43 +34,35 @@ import org.junit.Ignore;
// disk (but, should run successfully). Best to run w/
// -Dtests.codec=Standard, and w/ plenty of RAM, eg:
//
-// ant compile-core compile-test
+// ant compile-test
//
-// java -server -Xmx2g -Xms2g -d64 -cp .:lib/junit-4.7.jar:./build/classes/test:./build/classes/java:./build/classes/demo -Dlucene.version=4.0-dev -Dtests.codec=Standard -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.Test2BTerms
+// java -server -Xmx2g -Xms2g -d64 -cp .:lib/junit-4.7.jar:./build/classes/test:./build/classes/java -Dlucene.version=4.0-dev -Dtests.directory=SimpleFSDirectory -Dtests.codec=Standard -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.Test2BTerms
//
public class Test2BTerms extends LuceneTestCase {
- private final static BytesRef bytes = new BytesRef(20);
+ private final static int TOKEN_LEN = 10;
+
+ private final static BytesRef bytes = new BytesRef(TOKEN_LEN);
private static final class MyTokenStream extends TokenStream {
private final int tokensPerDoc;
private int tokenCount;
+ private int byteUpto;
public MyTokenStream(int tokensPerDoc) {
super(new MyAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY));
this.tokensPerDoc = tokensPerDoc;
addAttribute(TermToBytesRefAttribute.class);
+ bytes.length = TOKEN_LEN;
}
public boolean incrementToken() {
if (tokenCount >= tokensPerDoc) {
return false;
}
- final byte[] bs = bytes.bytes;
- for(int i=bytes.length-1;i>=0;i--) {
- int b = bs[i]&0xff;
- if (b == 0xff) {
- bs[i] = 0;
- } else {
- bs[i] = (byte) (++b);
- tokenCount++;
- return true;
- }
- }
- bytes.length++;
- bs[0] = 1;
+ random.nextBytes(bytes.bytes);
tokenCount++;
return true;
}
@@ -141,7 +133,7 @@ public class Test2BTerms extends LuceneT
Directory dir = FSDirectory.open(_TestUtil.getTempDir("2BTerms"));
IndexWriter w = new IndexWriter(dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())
.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
.setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()));
((LogMergePolicy) w.getConfig().getMergePolicy()).setUseCompoundFile(false);
@@ -156,14 +148,19 @@ public class Test2BTerms extends LuceneT
//w.setInfoStream(System.out);
final int numDocs = (int) (TERM_COUNT/TERMS_PER_DOC);
for(int i=0;i<numDocs;i++) {
+ final long t0 = System.currentTimeMillis();
w.addDocument(doc);
- System.out.println(i + " of " + numDocs);
+ System.out.println(i + " of " + numDocs + " " + (System.currentTimeMillis()-t0) + " msec");
}
System.out.println("now optimize...");
w.optimize();
w.close();
- _TestUtil.checkIndex(dir);
+ System.out.println("now CheckIndex...");
+ CheckIndex.Status status = _TestUtil.checkIndex(dir);
+ final long tc = status.segmentInfos.get(0).termIndexStatus.termCount;
+ assertTrue("count " + tc + " is not > " + Integer.MAX_VALUE, tc > Integer.MAX_VALUE);
+
dir.close();
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1005356&r1=1005355&r2=1005356&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Thu Oct 7 07:59:16 2010
@@ -77,7 +77,6 @@ import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Bits;
-import org.junit.Assume;
public class TestIndexWriter extends LuceneTestCase {
@@ -3532,7 +3531,7 @@ public class TestIndexWriter extends Luc
assertEquals(0, tps.nextPosition());
w.close();
- assertTrue(_TestUtil.checkIndex(dir));
+ _TestUtil.checkIndex(dir);
s.close();
dir.close();
}
@@ -5275,7 +5274,6 @@ public class TestIndexWriter extends Luc
// LUCENE-2593
public void testCorruptionAfterDiskFullDuringMerge() throws IOException {
MockDirectoryWrapper dir = newDirectory();
- final Random rand = random;
//IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setReaderPooling(true));
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergeScheduler(new SerialMergeScheduler()).setReaderPooling(true));
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java?rev=1005356&r1=1005355&r2=1005356&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/_TestUtil.java Thu Oct 7 07:59:16 2010
@@ -68,7 +68,7 @@ public class _TestUtil {
/** This runs the CheckIndex tool on the index in. If any
* issues are hit, a RuntimeException is thrown; else,
* true is returned. */
- public static boolean checkIndex(Directory dir) throws IOException {
+ public static CheckIndex.Status checkIndex(Directory dir) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
@@ -78,8 +78,9 @@ public class _TestUtil {
System.out.println("CheckIndex failed");
System.out.println(bos.toString());
throw new RuntimeException("CheckIndex failed");
- } else
- return true;
+ } else {
+ return indexStatus;
+ }
}
/** start and end are BOTH inclusive */