You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/12/19 01:24:06 UTC
svn commit: r1050738 [3/3] - in /lucene/dev/branches/bulkpostings: ./
lucene/ lucene/contrib/ant/src/java/org/apache/lucene/ant/
lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/
lucene/contrib/wordnet/src/java/org/apache/lucene/wo...
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Sun Dec 19 00:24:04 2010
@@ -248,8 +248,35 @@ public class MMapDirectory extends FSDir
throw new IOException("read past EOF");
}
}
+
+ @Override
+ public short readShort() throws IOException {
+ try {
+ return buffer.getShort();
+ } catch (BufferUnderflowException e) {
+ throw new IOException("read past EOF");
+ }
+ }
@Override
+ public int readInt() throws IOException {
+ try {
+ return buffer.getInt();
+ } catch (BufferUnderflowException e) {
+ throw new IOException("read past EOF");
+ }
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ try {
+ return buffer.getLong();
+ } catch (BufferUnderflowException e) {
+ throw new IOException("read past EOF");
+ }
+ }
+
+ @Override
public long getFilePointer() {
return buffer.position();
}
@@ -301,7 +328,6 @@ public class MMapDirectory extends FSDir
private final int maxBufSize;
private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
- private int curAvail; // redundant for speed: (bufSizes[curBufIndex] - curBuf.position())
private boolean isClone = false;
@@ -345,38 +371,67 @@ public class MMapDirectory extends FSDir
@Override
public byte readByte() throws IOException {
- // Performance might be improved by reading ahead into an array of
- // e.g. 128 bytes and readByte() from there.
- if (curAvail == 0) {
+ try {
+ return curBuf.get();
+ } catch (BufferUnderflowException e) {
curBufIndex++;
if (curBufIndex >= buffers.length)
throw new IOException("read past EOF");
curBuf = buffers[curBufIndex];
curBuf.position(0);
- curAvail = bufSizes[curBufIndex];
+ return curBuf.get();
}
- curAvail--;
- return curBuf.get();
}
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
- while (len > curAvail) {
- curBuf.get(b, offset, curAvail);
- len -= curAvail;
- offset += curAvail;
- curBufIndex++;
- if (curBufIndex >= buffers.length)
- throw new IOException("read past EOF");
- curBuf = buffers[curBufIndex];
- curBuf.position(0);
- curAvail = bufSizes[curBufIndex];
+ try {
+ curBuf.get(b, offset, len);
+ } catch (BufferUnderflowException e) {
+ int curAvail = curBuf.remaining();
+ while (len > curAvail) {
+ curBuf.get(b, offset, curAvail);
+ len -= curAvail;
+ offset += curAvail;
+ curBufIndex++;
+ if (curBufIndex >= buffers.length)
+ throw new IOException("read past EOF");
+ curBuf = buffers[curBufIndex];
+ curBuf.position(0);
+ curAvail = curBuf.remaining();
+ }
+ curBuf.get(b, offset, len);
}
- curBuf.get(b, offset, len);
- curAvail -= len;
}
@Override
+ public short readShort() throws IOException {
+ try {
+ return curBuf.getShort();
+ } catch (BufferUnderflowException e) {
+ return super.readShort();
+ }
+ }
+
+ @Override
+ public int readInt() throws IOException {
+ try {
+ return curBuf.getInt();
+ } catch (BufferUnderflowException e) {
+ return super.readInt();
+ }
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ try {
+ return curBuf.getLong();
+ } catch (BufferUnderflowException e) {
+ return super.readLong();
+ }
+ }
+
+ @Override
public long getFilePointer() {
return ((long) curBufIndex * maxBufSize) + curBuf.position();
}
@@ -387,7 +442,6 @@ public class MMapDirectory extends FSDir
curBuf = buffers[curBufIndex];
int bufOffset = (int) (pos - ((long) curBufIndex * maxBufSize));
curBuf.position(bufOffset);
- curAvail = bufSizes[curBufIndex] - bufOffset;
}
@Override
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/IOUtils.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/IOUtils.java Sun Dec 19 00:24:04 2010
@@ -64,4 +64,27 @@ public final class IOUtils {
else if (firstIOE != null)
throw firstIOE;
}
+
+ /**
+ * <p>Closes all given <tt>Closeable</tt>s, suppressing all thrown exceptions. Some of the <tt>Closeable</tt>s
+ * may be null, they are ignored. After everything is closed, method either throws the first of suppressed exceptions,
+ * or completes normally.</p>
+ * @param objects objects to call <tt>close()</tt> on
+ */
+ public static void closeSafely(Closeable... objects) throws IOException {
+ IOException firstIOE = null;
+
+ for (Closeable object : objects) {
+ try {
+ if (object != null)
+ object.close();
+ } catch (IOException ioe) {
+ if (firstIOE == null)
+ firstIOE = ioe;
+ }
+ }
+
+ if (firstIOE != null)
+ throw firstIOE;
+ }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java Sun Dec 19 00:24:04 2010
@@ -105,6 +105,5 @@ public class TestMergeSchedulerExternal
assertTrue(mergeCalled);
assertTrue(excCalled);
dir.close();
- assertTrue(ConcurrentMergeScheduler.anyUnhandledExceptions());
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearch.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearch.java Sun Dec 19 00:24:04 2010
@@ -17,7 +17,6 @@ package org.apache.lucene;
* limitations under the License.
*/
-import java.util.GregorianCalendar;
import java.util.Random;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -77,7 +76,6 @@ public class TestSearch extends LuceneTe
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
IndexWriter writer = new IndexWriter(directory, conf);
String[] docs = {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java Sun Dec 19 00:24:04 2010
@@ -82,7 +82,6 @@ public class TestSearchForDuplicates ext
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFiles);
- lmp.setUseCompoundDocStore(useCompoundFiles);
IndexWriter writer = new IndexWriter(directory, conf);
if (VERBOSE) {
System.out.println("TEST: now build index");
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Sun Dec 19 00:24:04 2010
@@ -570,7 +570,6 @@ public class TestAddIndexes extends Luce
Directory dir = newDirectory();
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
lmp.setUseCompoundFile(false);
- lmp.setUseCompoundDocStore(false);
lmp.setMergeFactor(100);
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer())
@@ -599,7 +598,6 @@ public class TestAddIndexes extends Luce
lmp = new LogByteSizeMergePolicy();
lmp.setMinMergeMB(0.0001);
lmp.setUseCompoundFile(false);
- lmp.setUseCompoundDocStore(false);
lmp.setMergeFactor(4);
writer = new IndexWriter(dir2, newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer())
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sun Dec 19 00:24:04 2010
@@ -293,6 +293,9 @@ public class TestBackwardsCompatibility
public void testIndexOldIndex() throws IOException {
for(int i=0;i<oldNames.length;i++) {
+ if (VERBOSE) {
+ System.out.println("TEST: oldName=" + oldNames[i]);
+ }
unzip(getDataFile("index." + oldNames[i] + ".zip"), oldNames[i]);
changeIndexWithAdds(random, oldNames[i]);
rmDir(oldNames[i]);
@@ -344,6 +347,10 @@ public class TestBackwardsCompatibility
f = d.getField("fie\u2C77ld");
assertEquals("field with non-ascii name", f.stringValue());
}
+
+ TermFreqVector tfv = reader.getTermFreqVector(i, "utf8");
+ assertNotNull("docID=" + i + " index=" + dirName, tfv);
+ assertTrue(tfv instanceof TermPositionVector);
} else
// Only ID 7 is deleted
assertEquals(7, i);
@@ -382,6 +389,7 @@ public class TestBackwardsCompatibility
Directory dir = newFSDirectory(new File(dirName));
// open writer
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
+ writer.setInfoStream(VERBOSE ? System.out : null);
// add 10 docs
for(int i=0;i<10;i++) {
addDoc(writer, 35+i);
@@ -496,7 +504,6 @@ public class TestBackwardsCompatibility
Directory dir = newFSDirectory(new File(dirName));
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10);
((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
- ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(doCFS);
IndexWriter writer = new IndexWriter(dir, conf);
for(int i=0;i<35;i++) {
@@ -508,7 +515,6 @@ public class TestBackwardsCompatibility
// open fresh writer so we get no prx file in the added segment
conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(10);
((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
- ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(doCFS);
writer = new IndexWriter(dir, conf);
addNoProxDoc(writer);
writer.close();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java Sun Dec 19 00:24:04 2010
@@ -270,7 +270,7 @@ public class TestCodecs extends LuceneTe
final Directory dir = newDirectory();
this.write(fieldInfos, dir, fields);
- final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+ final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
si.setHasProx(false);
final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
@@ -318,7 +318,7 @@ public class TestCodecs extends LuceneTe
final Directory dir = newDirectory();
this.write(fieldInfos, dir, fields);
- final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
+ final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()), fieldInfos.hasVectors());
final FieldsProducer terms = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
@@ -607,7 +607,7 @@ public class TestCodecs extends LuceneTe
final int termIndexInterval = this.nextInt(13, 27);
final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, CodecProvider.getDefault());
- final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval, codecInfo);
+ final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, 10000, termIndexInterval, codecInfo);
final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
Arrays.sort(fields);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Sun Dec 19 00:24:04 2010
@@ -96,7 +96,7 @@ public class TestConcurrentMergeSchedule
writer.addDocument(doc);
failure.setDoFail();
try {
- writer.flush(true, false, true);
+ writer.flush(true, true);
if (failure.hitExc) {
fail("failed to hit IOException");
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Sun Dec 19 00:24:04 2010
@@ -195,16 +195,13 @@ public class TestDeletionPolicy extends
final double SECONDS = 2.0;
- boolean useCompoundFile = true;
-
Directory dir = newDirectory();
ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer())
.setIndexDeletionPolicy(policy);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
+ lmp.setUseCompoundFile(true);
IndexWriter writer = new IndexWriter(dir, conf);
writer.close();
@@ -219,8 +216,7 @@ public class TestDeletionPolicy extends
new MockAnalyzer()).setOpenMode(
OpenMode.APPEND).setIndexDeletionPolicy(policy);
lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
+ lmp.setUseCompoundFile(true);
writer = new IndexWriter(dir, conf);
for(int j=0;j<17;j++) {
addDoc(writer);
@@ -298,7 +294,6 @@ public class TestDeletionPolicy extends
.setMergeScheduler(new SerialMergeScheduler());
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
lmp.setMergeFactor(10);
IndexWriter writer = new IndexWriter(dir, conf);
for(int i=0;i<107;i++) {
@@ -311,7 +306,6 @@ public class TestDeletionPolicy extends
OpenMode.APPEND).setIndexDeletionPolicy(policy);
lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
writer = new IndexWriter(dir, conf);
writer.optimize();
writer.close();
@@ -488,7 +482,6 @@ public class TestDeletionPolicy extends
.setMaxBufferedDocs(10);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
IndexWriter writer = new IndexWriter(dir, conf);
for(int i=0;i<107;i++) {
addDoc(writer);
@@ -499,7 +492,6 @@ public class TestDeletionPolicy extends
.setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
writer = new IndexWriter(dir, conf);
writer.optimize();
writer.close();
@@ -539,7 +531,6 @@ public class TestDeletionPolicy extends
.setMaxBufferedDocs(10);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
IndexWriter writer = new IndexWriter(dir, conf);
for(int i=0;i<17;i++) {
addDoc(writer);
@@ -597,7 +588,6 @@ public class TestDeletionPolicy extends
.setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
IndexWriter writer = new IndexWriter(dir, conf);
writer.close();
Term searchTerm = new Term("content", "aaa");
@@ -609,7 +599,6 @@ public class TestDeletionPolicy extends
.setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
writer = new IndexWriter(dir, conf);
for(int j=0;j<17;j++) {
addDoc(writer);
@@ -630,7 +619,6 @@ public class TestDeletionPolicy extends
.setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
writer = new IndexWriter(dir, conf);
writer.optimize();
// this is a commit
@@ -706,7 +694,6 @@ public class TestDeletionPolicy extends
.setMaxBufferedDocs(10);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
IndexWriter writer = new IndexWriter(dir, conf);
writer.close();
Term searchTerm = new Term("content", "aaa");
@@ -720,7 +707,6 @@ public class TestDeletionPolicy extends
.setMaxBufferedDocs(10);
lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundFile);
- lmp.setUseCompoundDocStore(useCompoundFile);
writer = new IndexWriter(dir, conf);
for(int j=0;j<17;j++) {
addDoc(writer);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestDoc.java Sun Dec 19 00:24:04 2010
@@ -192,7 +192,7 @@ public class TestDoc extends LuceneTestC
SegmentReader r1 = SegmentReader.get(true, si1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
SegmentReader r2 = SegmentReader.get(true, si2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
- SegmentMerger merger = new SegmentMerger(si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, null, CodecProvider.getDefault(), null);
+ SegmentMerger merger = new SegmentMerger(si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, null, CodecProvider.getDefault(), null, new FieldInfos());
merger.add(r1);
merger.add(r2);
@@ -201,8 +201,8 @@ public class TestDoc extends LuceneTestC
r2.close();
final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
- useCompoundFile, -1, null, false, merger.hasProx(), merger.getSegmentCodecs(),
- merger.hasVectors());
+ useCompoundFile, merger.fieldInfos().hasProx(), merger.getSegmentCodecs(),
+ merger.fieldInfos().hasVectors());
if (useCompoundFile) {
Collection<String> filesToDelete = merger.createCompoundFile(merged + ".cfs", info);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Sun Dec 19 00:24:04 2010
@@ -53,7 +53,6 @@ public class TestFieldsReader extends Lu
dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false);
- ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundDocStore(false);
IndexWriter writer = new IndexWriter(dir, conf);
writer.addDocument(testDoc);
writer.close();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Sun Dec 19 00:24:04 2010
@@ -56,7 +56,6 @@ public class TestIndexFileDeleter extend
addDoc(writer, i);
}
mergePolicy.setUseCompoundFile(false);
- mergePolicy.setUseCompoundDocStore(false);
for(;i<45;i++) {
addDoc(writer, i);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Sun Dec 19 00:24:04 2010
@@ -240,7 +240,6 @@ public class TestIndexReaderCloneNorms e
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(true);
- lmp.setUseCompoundDocStore(true);
iw.close();
}
@@ -293,7 +292,6 @@ public class TestIndexReaderCloneNorms e
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(compound);
- lmp.setUseCompoundDocStore(compound);
IndexWriter iw = new IndexWriter(dir, conf);
for (int i = 0; i < ndocs; i++) {
iw.addDocument(newDoc());
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Dec 19 00:24:04 2010
@@ -157,7 +157,7 @@ public class TestIndexWriter extends Luc
String[] startFiles = dir.listAll();
SegmentInfos infos = new SegmentInfos();
infos.read(dir);
- new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null, CodecProvider.getDefault());
+ new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, CodecProvider.getDefault());
String[] endFiles = dir.listAll();
Arrays.sort(startFiles);
@@ -866,7 +866,7 @@ public class TestIndexWriter extends Luc
doc.add(newField("field", "aaa", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
for(int i=0;i<19;i++)
writer.addDocument(doc);
- writer.flush(false, true, true);
+ writer.flush(false, true);
writer.close();
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
@@ -1039,7 +1039,6 @@ public class TestIndexWriter extends Luc
LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
lmp.setMergeFactor(2);
lmp.setUseCompoundFile(false);
- lmp.setUseCompoundDocStore(false);
Document doc = new Document();
String contents = "aa bb cc dd ee ff gg hh ii jj kk";
@@ -1075,7 +1074,6 @@ public class TestIndexWriter extends Luc
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()));
LogMergePolicy lmp2 = (LogMergePolicy) writer.getConfig().getMergePolicy();
lmp2.setUseCompoundFile(false);
- lmp2.setUseCompoundDocStore(false);
writer.optimize();
writer.close();
}
@@ -2257,7 +2255,6 @@ public class TestIndexWriter extends Luc
TEST_VERSION_CURRENT, new MockAnalyzer()));
LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
lmp.setUseCompoundFile(false);
- lmp.setUseCompoundDocStore(false);
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
writer.setInfoStream(new PrintStream(bos));
writer.addDocument(new Document());
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Sun Dec 19 00:24:04 2010
@@ -689,7 +689,6 @@ public class TestIndexWriterDelete exten
LogMergePolicy lmp = (LogMergePolicy) modifier.getConfig().getMergePolicy();
lmp.setUseCompoundFile(true);
- lmp.setUseCompoundDocStore(true);
dir.failOn(failure.reset());
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java Sun Dec 19 00:24:04 2010
@@ -60,7 +60,7 @@ public class TestIndexWriterMerging exte
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
setMergePolicy(newLogMergePolicy(2))
);
-
+ writer.setInfoStream(VERBOSE ? System.out : null);
writer.addIndexes(indexA, indexB);
writer.optimize();
writer.close();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Sun Dec 19 00:24:04 2010
@@ -209,7 +209,7 @@ public class TestIndexWriterOnDiskFull e
System.out.println("TEST: iter=" + iter);
// Start with 100 bytes more than we are currently using:
- long diskFree = diskUsage+100;
+ long diskFree = diskUsage+_TestUtil.nextInt(random, 50, 200);
int method = iter;
@@ -226,12 +226,16 @@ public class TestIndexWriterOnDiskFull e
}
while(!done) {
+ if (VERBOSE) {
+ System.out.println("TEST: cycle...");
+ }
// Make a new dir that will enforce disk usage:
MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
IOException err = null;
-
+ writer.setInfoStream(VERBOSE ? System.out : null);
+
MergeScheduler ms = writer.getConfig().getMergeScheduler();
for(int x=0;x<2;x++) {
if (ms instanceof ConcurrentMergeScheduler)
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Sun Dec 19 00:24:04 2010
@@ -237,7 +237,7 @@ public class TestIndexWriterReader exten
writer.setInfoStream(infoStream);
// create the index
createIndexNoClose(!optimize, "index1", writer);
- writer.flush(false, true, true);
+ writer.flush(false, true);
// create a 2nd index
Directory dir2 = newDirectory();
@@ -317,7 +317,7 @@ public class TestIndexWriterReader exten
writer.setInfoStream(infoStream);
// create the index
createIndexNoClose(!optimize, "index1", writer);
- writer.flush(false, true, true);
+ writer.flush(false, true);
// get a reader
IndexReader r1 = writer.getReader();
@@ -524,7 +524,7 @@ public class TestIndexWriterReader exten
IndexReader r1 = writer.getReader();
assertEquals(0, r1.maxDoc());
createIndexNoClose(false, "index1", writer);
- writer.flush(!optimize, true, true);
+ writer.flush(!optimize, true);
IndexReader iwr1 = writer.getReader();
assertEquals(100, iwr1.maxDoc());
@@ -536,7 +536,7 @@ public class TestIndexWriterReader exten
Document d = createDocument(x, "index1", 5);
writer.addDocument(d);
}
- writer.flush(false, true, true);
+ writer.flush(false, true);
// verify the reader was reopened internally
IndexReader iwr2 = writer.getReader();
assertTrue(iwr2 != r1);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java Sun Dec 19 00:24:04 2010
@@ -358,47 +358,6 @@ public class TestIndexWriterWithThreads
_testMultipleThreadsFailure(new FailOnlyOnAbortOrFlush(true));
}
- // Throws IOException during DocumentsWriter.closeDocStore
- private static class FailOnlyInCloseDocStore extends MockDirectoryWrapper.Failure {
- private boolean onlyOnce;
- public FailOnlyInCloseDocStore(boolean onlyOnce) {
- this.onlyOnce = onlyOnce;
- }
- @Override
- public void eval(MockDirectoryWrapper dir) throws IOException {
- if (doFail) {
- StackTraceElement[] trace = new Exception().getStackTrace();
- for (int i = 0; i < trace.length; i++) {
- if ("closeDocStore".equals(trace[i].getMethodName())) {
- if (onlyOnce)
- doFail = false;
- throw new IOException("now failing on purpose");
- }
- }
- }
- }
- }
-
- // LUCENE-1130: test IOException in closeDocStore
- public void testIOExceptionDuringCloseDocStore() throws IOException {
- _testSingleThreadFailure(new FailOnlyInCloseDocStore(false));
- }
-
- // LUCENE-1130: test IOException in closeDocStore
- public void testIOExceptionDuringCloseDocStoreOnlyOnce() throws IOException {
- _testSingleThreadFailure(new FailOnlyInCloseDocStore(true));
- }
-
- // LUCENE-1130: test IOException in closeDocStore, with threads
- public void testIOExceptionDuringCloseDocStoreWithThreads() throws Exception {
- _testMultipleThreadsFailure(new FailOnlyInCloseDocStore(false));
- }
-
- // LUCENE-1130: test IOException in closeDocStore, with threads
- public void testIOExceptionDuringCloseDocStoreWithThreadsOnlyOnce() throws Exception {
- _testMultipleThreadsFailure(new FailOnlyInCloseDocStore(true));
- }
-
// Throws IOException during DocumentsWriter.writeSegment
private static class FailOnlyInWriteSegment extends MockDirectoryWrapper.Failure {
private boolean onlyOnce;
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestLazyBug.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestLazyBug.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestLazyBug.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestLazyBug.java Sun Dec 19 00:24:04 2010
@@ -66,8 +66,7 @@ public class TestLazyBug extends LuceneT
TEST_VERSION_CURRENT, new MockAnalyzer()));
LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
lmp.setUseCompoundFile(false);
- lmp.setUseCompoundDocStore(false);
-
+
for (int d = 1; d <= NUM_DOCS; d++) {
Document doc = new Document();
for (int f = 1; f <= NUM_FIELDS; f++ ) {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Sun Dec 19 00:24:04 2010
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -34,6 +35,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Bits;
@@ -113,6 +115,9 @@ public class TestNRTThreads extends Luce
final AtomicBoolean failed = new AtomicBoolean();
final AtomicInteger addCount = new AtomicInteger();
final AtomicInteger delCount = new AtomicInteger();
+
+ final List<String> delIDs = Collections.synchronizedList(new ArrayList<String>());
+
final long stopTime = System.currentTimeMillis() + RUN_TIME_SEC*1000;
Thread[] threads = new Thread[NUM_INDEX_THREADS];
for(int thread=0;thread<NUM_INDEX_THREADS;thread++) {
@@ -152,7 +157,11 @@ public class TestNRTThreads extends Luce
for(String id : toDeleteIDs) {
writer.deleteDocuments(new Term("id", id));
}
- delCount.addAndGet(toDeleteIDs.size());
+ final int count = delCount.addAndGet(toDeleteIDs.size());
+ if (VERBOSE) {
+ System.out.println(Thread.currentThread().getName() + ": tot " + count + " deletes");
+ }
+ delIDs.addAll(toDeleteIDs);
toDeleteIDs.clear();
}
addCount.getAndIncrement();
@@ -163,6 +172,9 @@ public class TestNRTThreads extends Luce
throw new RuntimeException(exc);
}
}
+ if (VERBOSE) {
+ System.out.println(Thread.currentThread().getName() + ": indexing done");
+ }
}
};
threads[thread].setDaemon(true);
@@ -315,8 +327,20 @@ public class TestNRTThreads extends Luce
if (VERBOSE) {
System.out.println("TEST: done join [" + (System.currentTimeMillis()-t0) + " ms]; addCount=" + addCount + " delCount=" + delCount);
}
+
+ final IndexReader r2 = writer.getReader();
+ final IndexSearcher s = new IndexSearcher(r2);
+ for(String id : delIDs) {
+ final TopDocs hits = s.search(new TermQuery(new Term("id", id)), 1);
+ if (hits.totalHits != 0) {
+ fail("doc id=" + id + " is supposed to be deleted, but got docID=" + hits.scoreDocs[0].doc);
+ }
+ }
+ assertEquals("index=" + writer.segString() + " addCount=" + addCount + " delCount=" + delCount, addCount.get() - delCount.get(), r2.numDocs());
+ r2.close();
+
writer.commit();
- assertEquals("index=" + writer.segString(), addCount.get() - delCount.get(), writer.numDocs());
+ assertEquals("index=" + writer.segString() + " addCount=" + addCount + " delCount=" + delCount, addCount.get() - delCount.get(), writer.numDocs());
writer.close(false);
dir.close();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNoMergePolicy.java Sun Dec 19 00:24:04 2010
@@ -35,16 +35,13 @@ public class TestNoMergePolicy extends L
assertNull(mp.findMerges(null));
assertNull(mp.findMergesForOptimize(null, 0, null));
assertNull(mp.findMergesToExpungeDeletes(null));
- assertFalse(mp.useCompoundDocStore(null));
assertFalse(mp.useCompoundFile(null, null));
mp.close();
}
@Test
public void testCompoundFiles() throws Exception {
- assertFalse(NoMergePolicy.NO_COMPOUND_FILES.useCompoundDocStore(null));
assertFalse(NoMergePolicy.NO_COMPOUND_FILES.useCompoundFile(null, null));
- assertTrue(NoMergePolicy.COMPOUND_FILES.useCompoundDocStore(null));
assertTrue(NoMergePolicy.COMPOUND_FILES.useCompoundFile(null, null));
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestNorms.java Sun Dec 19 00:24:04 2010
@@ -154,7 +154,6 @@ public class TestNorms extends LuceneTes
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(true);
- lmp.setUseCompoundDocStore(true);
iw.close();
}
@@ -199,7 +198,6 @@ public class TestNorms extends LuceneTes
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(compound);
- lmp.setUseCompoundDocStore(compound);
for (int i = 0; i < ndocs; i++) {
iw.addDocument(newDoc());
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Sun Dec 19 00:24:04 2010
@@ -219,7 +219,6 @@ public class TestOmitTf extends LuceneTe
LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
lmp.setMergeFactor(2);
lmp.setUseCompoundFile(false);
- lmp.setUseCompoundDocStore(false);
Document d = new Document();
Field f1 = newField("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java Sun Dec 19 00:24:04 2010
@@ -137,7 +137,6 @@ public class TestPerFieldCodecSupport ex
iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())
.setOpenMode(OpenMode.APPEND).setCodecProvider(provider);
((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundFile(false);
- ((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundDocStore(false);
((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10);
iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
@@ -294,4 +293,4 @@ public class TestPerFieldCodecSupport ex
}
dir.close();
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java Sun Dec 19 00:24:04 2010
@@ -24,7 +24,6 @@ import java.util.Random;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
@@ -73,7 +72,7 @@ public class TestPerSegmentDeletes exten
// flushing without applying deletes means
// there will still be deletes in the segment infos
- writer.flush(false, false, false);
+ writer.flush(false, false);
assertTrue(writer.bufferedDeletes.any());
// get reader flushes pending deletes
@@ -86,7 +85,7 @@ public class TestPerSegmentDeletes exten
// merge segments 0 and 1
// which should apply the delete id:2
writer.deleteDocuments(new Term("id", "2"));
- writer.flush(false, false, false);
+ writer.flush(false, false);
fsmp.doMerge = true;
fsmp.start = 0;
fsmp.length = 2;
@@ -179,12 +178,12 @@ public class TestPerSegmentDeletes exten
writer.addDocument(TestIndexWriterReader.createDocument(x, "5", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
- writer.flush(false, true, false);
+ writer.flush(false, false);
for (int x = 25; x < 30; x++) {
writer.addDocument(TestIndexWriterReader.createDocument(x, "5", 2));
//System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
}
- writer.flush(false, true, false);
+ writer.flush(false, false);
//System.out.println("infos3:"+writer.segmentInfos);
@@ -287,11 +286,6 @@ public class TestPerSegmentDeletes exten
}
@Override
- public boolean useCompoundDocStore(SegmentInfos segments) {
- return useCompoundFile;
- }
-
- @Override
public boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment) {
return useCompoundFile;
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Sun Dec 19 00:24:04 2010
@@ -73,14 +73,14 @@ public class TestSegmentMerger extends L
}
public void testMerge() throws IOException {
- SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, CodecProvider.getDefault(), null);
+ SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, CodecProvider.getDefault(), null, new FieldInfos());
merger.add(reader1);
merger.add(reader2);
int docsMerged = merger.merge();
assertTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
- SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, -1,
- null, false, merger.hasProx(), merger.getSegmentCodecs(), merger.hasVectors()),
+ SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, merger.fieldInfos().hasProx(),
+ merger.getSegmentCodecs(), merger.fieldInfos().hasVectors()),
BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
assertTrue(mergedReader != null);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java Sun Dec 19 00:24:04 2010
@@ -154,7 +154,6 @@ public class TestStressIndexing2 extends
w.commit();
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setUseCompoundFile(false);
- lmp.setUseCompoundDocStore(false);
lmp.setMergeFactor(mergeFactor);
/***
w.setMaxMergeDocs(Integer.MAX_VALUE);
@@ -211,7 +210,6 @@ public class TestStressIndexing2 extends
w.setInfoStream(VERBOSE ? System.out : null);
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setUseCompoundFile(false);
- lmp.setUseCompoundDocStore(false);
lmp.setMergeFactor(mergeFactor);
threads = new IndexingThread[nThreads];
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java Sun Dec 19 00:24:04 2010
@@ -277,6 +277,7 @@ public class TestMultiSearcher extends L
indexSearcher2.close();
ramDirectory1.close();
ramDirectory2.close();
+ searcher.close();
}
/* uncomment this when the highest score is always normalized to 1.0, even when it was < 1.0
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestParallelMultiSearcher.java Sun Dec 19 00:24:04 2010
@@ -18,16 +18,34 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.util._TestUtil;
/**
* Unit tests for the ParallelMultiSearcher
*/
public class TestParallelMultiSearcher extends TestMultiSearcher {
+ List<ExecutorService> pools = new ArrayList<ExecutorService>();
+
+ @Override
+ public void tearDown() throws Exception {
+ for (ExecutorService exec : pools)
+ exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
+ pools.clear();
+ super.tearDown();
+ }
@Override
protected MultiSearcher getMultiSearcherInstance(Searcher[] searchers)
throws IOException {
- return new ParallelMultiSearcher(searchers);
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+ pools.add(exec);
+ return new ParallelMultiSearcher(exec, searchers);
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestSort.java Sun Dec 19 00:24:04 2010
@@ -25,6 +25,10 @@ import java.util.BitSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -51,6 +55,7 @@ import org.apache.lucene.store.LockObtai
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.DocIdBitSet;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
/**
* Unit tests for sorting code.
@@ -611,7 +616,8 @@ public class TestSort extends LuceneTest
// Don't close the multiSearcher. it would close the full searcher too!
// Do the same for a ParallelMultiSearcher
- Searcher parallelSearcher=new ParallelMultiSearcher (full);
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+ Searcher parallelSearcher=new ParallelMultiSearcher (exec, full);
sort.setSort (new SortField ("int", SortField.INT),
new SortField ("string", SortField.STRING),
@@ -622,7 +628,8 @@ public class TestSort extends LuceneTest
new SortField ("string", SortField.STRING),
new SortField ("float", SortField.FLOAT, true) );
assertMatches (parallelSearcher, queryG, sort, "ZYXW");
- // Don't close the parallelSearcher. it would close the full searcher too!
+ parallelSearcher.close();
+ exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
}
// test sorts using a series of fields
@@ -690,8 +697,11 @@ public class TestSort extends LuceneTest
// test a variety of sorts using a parallel multisearcher
public void testParallelMultiSort() throws Exception {
- Searcher searcher = new ParallelMultiSearcher (searchX, searchY);
+ ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random, 2, 8));
+ Searcher searcher = new ParallelMultiSearcher (exec, searchX, searchY);
runMultiSorts(searcher, false);
+ searcher.close();
+ exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
}
// test that the relevancy scores are the same even if
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/LuceneTestCase.java Sun Dec 19 00:24:04 2010
@@ -336,6 +336,11 @@ public abstract class LuceneTestCase ext
@AfterClass
public static void afterClassLuceneTestCaseJ4() {
+ int rogueThreads = threadCleanup("test class");
+ if (rogueThreads > 0) {
+ // TODO: fail here once the leaks are fixed.
+ System.err.println("RESOURCE LEAK: test class left " + rogueThreads + " thread(s) running");
+ }
String codecDescription;
CodecProvider cp = CodecProvider.getDefault();
@@ -434,13 +439,13 @@ public abstract class LuceneTestCase ext
savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler();
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
public void uncaughtException(Thread t, Throwable e) {
+ testsFailed = true;
uncaughtExceptions.add(new UncaughtExceptionEntry(t, e));
if (savedUncaughtExceptionHandler != null)
savedUncaughtExceptionHandler.uncaughtException(t, e);
}
});
- ConcurrentMergeScheduler.setTestMode();
savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount();
}
@@ -469,6 +474,18 @@ public abstract class LuceneTestCase ext
assertTrue("ensure your setUp() calls super.setUp()!!!", setup);
setup = false;
BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount);
+ if (!getClass().getName().startsWith("org.apache.solr")) {
+ int rogueThreads = threadCleanup("test method: '" + getName() + "'");
+ if (rogueThreads > 0) {
+ System.err.println("RESOURCE LEAK: test method: '" + getName()
+ + "' left " + rogueThreads + " thread(s) running");
+ // TODO: fail, but print seed for now.
+ if (!testsFailed && uncaughtExceptions.isEmpty()) {
+ reportAdditionalFailureInfo();
+ }
+ }
+ }
+ Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler);
try {
if (!uncaughtExceptions.isEmpty()) {
@@ -496,19 +513,61 @@ public abstract class LuceneTestCase ext
// isolated in distinct test methods
assertSaneFieldCaches(getTestLabel());
- if (ConcurrentMergeScheduler.anyUnhandledExceptions()) {
- // Clear the failure so that we don't just keep
- // failing subsequent test cases
- ConcurrentMergeScheduler.clearUnhandledExceptions();
- fail("ConcurrentMergeScheduler hit unhandled exceptions");
- }
} finally {
purgeFieldCache(FieldCache.DEFAULT);
}
-
- Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler);
}
+ private final static int THREAD_STOP_GRACE_MSEC = 1000;
+ // jvm-wide list of 'rogue threads' we found, so they only get reported once.
+ private final static IdentityHashMap<Thread,Boolean> rogueThreads = new IdentityHashMap<Thread,Boolean>();
+
+ /**
+ * Looks for leftover running threads, trying to kill them off,
+ * so they don't fail future tests.
+ * returns the number of rogue threads that it found.
+ */
+ private static int threadCleanup(String context) {
+ // educated guess
+ Thread[] stillRunning = new Thread[Thread.activeCount()+1];
+ int threadCount = 0;
+ int rogueCount = 0;
+
+ if ((threadCount = Thread.enumerate(stillRunning)) > 1) {
+ while (threadCount == stillRunning.length) {
+ // truncated response
+ stillRunning = new Thread[stillRunning.length*2];
+ threadCount = Thread.enumerate(stillRunning);
+ }
+
+ for (int i = 0; i < threadCount; i++) {
+ Thread t = stillRunning[i];
+
+ if (t.isAlive() &&
+ !rogueThreads.containsKey(t) &&
+ t != Thread.currentThread() &&
+ // TODO: TimeLimitingCollector starts a thread statically.... WTF?!
+ !t.getName().equals("TimeLimitedCollector timer thread")) {
+ System.err.println("WARNING: " + context + " left thread running: " + t);
+ rogueThreads.put(t, true);
+ rogueCount++;
+ // wait on the thread to die of natural causes
+ try {
+ t.join(THREAD_STOP_GRACE_MSEC);
+ } catch (InterruptedException e) { e.printStackTrace(); }
+ // try to stop the thread:
+ t.setUncaughtExceptionHandler(null);
+ Thread.setDefaultUncaughtExceptionHandler(null);
+ t.interrupt();
+ try {
+ t.join(THREAD_STOP_GRACE_MSEC);
+ } catch (InterruptedException e) { e.printStackTrace(); }
+ }
+ }
+ }
+ return rogueCount;
+ }
+
/**
* Asserts that FieldCacheSanityChecker does not detect any
* problems with FieldCache.DEFAULT.
@@ -688,7 +747,6 @@ public abstract class LuceneTestCase ext
public static LogMergePolicy newLogMergePolicy(Random r) {
LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy();
- logmp.setUseCompoundDocStore(r.nextBoolean());
logmp.setUseCompoundFile(r.nextBoolean());
logmp.setCalibrateSizeByDeletes(r.nextBoolean());
if (r.nextInt(3) == 2) {
@@ -702,14 +760,12 @@ public abstract class LuceneTestCase ext
public static LogMergePolicy newLogMergePolicy(boolean useCFS) {
LogMergePolicy logmp = newLogMergePolicy();
logmp.setUseCompoundFile(useCFS);
- logmp.setUseCompoundDocStore(useCFS);
return logmp;
}
public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) {
LogMergePolicy logmp = newLogMergePolicy();
logmp.setUseCompoundFile(useCFS);
- logmp.setUseCompoundDocStore(useCFS);
logmp.setMergeFactor(mergeFactor);
return logmp;
}
@@ -773,6 +829,11 @@ public abstract class LuceneTestCase ext
} catch (ClassCastException e) {
// TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random
fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)];
+
+ if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store
+ fsdirClass = "org.apache.lucene.store." + fsdirClass;
+ }
+
clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class);
}
MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f, lf));
Modified: lucene/dev/branches/bulkpostings/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/CHANGES.txt?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/CHANGES.txt (original)
+++ lucene/dev/branches/bulkpostings/solr/CHANGES.txt Sun Dec 19 00:24:04 2010
@@ -314,6 +314,8 @@ New Features
Adding a parameter NOW=<time_in_ms> to the request will override the
current time. (Peter Sturge, yonik)
+* SOLR-2188: Added "maxTokenLength" argument to the factories for ClassicTokenizer,
+ StandardTokenizer, and UAX29URLEmailTokenizer. (Steven Rowe)
Optimizations
----------------------
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/ClassicTokenizerFactory.java Sun Dec 19 00:24:04 2010
@@ -19,6 +19,8 @@ package org.apache.solr.analysis;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.ClassicTokenizer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
import java.io.Reader;
import java.util.Map;
@@ -28,13 +30,20 @@ import java.util.Map;
*/
public class ClassicTokenizerFactory extends BaseTokenizerFactory {
+
+ private int maxTokenLength;
+
@Override
public void init(Map<String,String> args) {
super.init(args);
assureMatchVersion();
+ maxTokenLength = getInt("maxTokenLength",
+ StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
}
public Tokenizer create(Reader input) {
- return new ClassicTokenizer(luceneMatchVersion, input);
+ ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, input);
+ tokenizer.setMaxTokenLength(maxTokenLength);
+ return tokenizer;
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/StandardTokenizerFactory.java Sun Dec 19 00:24:04 2010
@@ -17,6 +17,7 @@
package org.apache.solr.analysis;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import java.io.Reader;
@@ -27,13 +28,21 @@ import java.util.Map;
*/
public class StandardTokenizerFactory extends BaseTokenizerFactory {
+
+ private int maxTokenLength;
+
@Override
public void init(Map<String,String> args) {
super.init(args);
assureMatchVersion();
+ maxTokenLength = getInt("maxTokenLength",
+ StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
}
public StandardTokenizer create(Reader input) {
- return new StandardTokenizer(luceneMatchVersion, input);
+ StandardTokenizer tokenizer
+ = new StandardTokenizer(luceneMatchVersion, input);
+ tokenizer.setMaxTokenLength(maxTokenLength);
+ return tokenizer;
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java Sun Dec 19 00:24:04 2010
@@ -20,6 +20,7 @@ package org.apache.solr.analysis;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
import java.io.Reader;
@@ -31,13 +32,20 @@ import java.util.Map;
*/
public class UAX29URLEmailTokenizerFactory extends BaseTokenizerFactory {
+
+ private int maxTokenLength;
+
@Override
public void init(Map<String,String> args) {
super.init(args);
assureMatchVersion();
+ maxTokenLength = getInt("maxTokenLength",
+ StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
}
public UAX29URLEmailTokenizer create(Reader input) {
- return new UAX29URLEmailTokenizer(input);
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(input);
+ tokenizer.setMaxTokenLength(maxTokenLength);
+ return tokenizer;
}
}
Modified: lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestStandardFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestStandardFactories.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestStandardFactories.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestStandardFactories.java Sun Dec 19 00:24:04 2010
@@ -19,6 +19,8 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -40,6 +42,24 @@ public class TestStandardFactories exten
new String[] {"Wha\u0301t's", "this", "thing", "do" });
}
+ public void testStandardTokenizerMaxTokenLength() throws Exception {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0 ; i < 100 ; ++i) {
+ builder.append("abcdefg"); // 7 * 100 = 700 char "word"
+ }
+ String longWord = builder.toString();
+ String content = "one two three " + longWord + " four five six";
+ Reader reader = new StringReader(content);
+ Map<String,String> args = new HashMap<String,String>();
+ args.put("luceneMatchVersion", DEFAULT_VERSION_PARAM.get("luceneMatchVersion"));
+ args.put("maxTokenLength", "1000");
+ StandardTokenizerFactory factory = new StandardTokenizerFactory();
+ factory.init(args);
+ Tokenizer stream = factory.create(reader);
+ assertTokenStreamContents(stream,
+ new String[] {"one", "two", "three", longWord, "four", "five", "six" });
+ }
+
/**
* Test ClassicTokenizerFactory
*/
@@ -52,6 +72,24 @@ public class TestStandardFactories exten
new String[] {"What's", "this", "thing", "do" });
}
+ public void testClassicTokenizerMaxTokenLength() throws Exception {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0 ; i < 100 ; ++i) {
+ builder.append("abcdefg"); // 7 * 100 = 700 char "word"
+ }
+ String longWord = builder.toString();
+ String content = "one two three " + longWord + " four five six";
+ Reader reader = new StringReader(content);
+ Map<String,String> args = new HashMap<String,String>();
+ args.put("luceneMatchVersion", DEFAULT_VERSION_PARAM.get("luceneMatchVersion"));
+ args.put("maxTokenLength", "1000");
+ ClassicTokenizerFactory factory = new ClassicTokenizerFactory();
+ factory.init(args);
+ Tokenizer stream = factory.create(reader);
+ assertTokenStreamContents(stream,
+ new String[] {"one", "two", "three", longWord, "four", "five", "six" });
+ }
+
/**
* Test ClassicFilterFactory
*/
Modified: lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java?rev=1050738&r1=1050737&r2=1050738&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/branches/bulkpostings/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java Sun Dec 19 00:24:04 2010
@@ -19,6 +19,9 @@ package org.apache.solr.analysis;
import java.io.Reader;
import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.lucene.analysis.Tokenizer;
/**
@@ -152,4 +155,22 @@ public class TestUAX29URLEmailTokenizerF
}
);
}
+
+ public void testMaxTokenLength() throws Exception {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0 ; i < 100 ; ++i) {
+ builder.append("abcdefg"); // 7 * 100 = 700 char "word"
+ }
+ String longWord = builder.toString();
+ String content = "one two three " + longWord + " four five six";
+ Reader reader = new StringReader(content);
+ Map<String,String> args = new HashMap<String,String>();
+ args.put("luceneMatchVersion", DEFAULT_VERSION_PARAM.get("luceneMatchVersion"));
+ args.put("maxTokenLength", "1000");
+ UAX29URLEmailTokenizerFactory factory = new UAX29URLEmailTokenizerFactory();
+ factory.init(args);
+ Tokenizer stream = factory.create(reader);
+ assertTokenStreamContents(stream,
+ new String[] {"one", "two", "three", longWord, "four", "five", "six" });
+ }
}