You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/10/27 22:49:14 UTC
svn commit: r1190017 - in /lucene/dev/trunk: ./
dev-tools/idea/lucene/contrib/ lucene/
lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/
lucene/src/java/org/apache/lucene/index/codecs/preflex/
lucene/src/java/org/apache/lucene/st...
Author: mikemccand
Date: Thu Oct 27 20:49:13 2011
New Revision: 1190017
URL: http://svn.apache.org/viewvc?rev=1190017&view=rev
Log:
LUCENE-2205: port to trunk in preflex codec; port TestPagedBytes and PagedBytes.DataInput/Output to trunk
Added:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java
- copied, changed from r1189914, lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermInfosReaderIndex.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java
- copied, changed from r1189914, lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermInfosReaderIndex.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestPagedBytes.java
- copied unchanged from r1189914, lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestPagedBytes.java
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/trunk/solr/ (props changed)
lucene/dev/trunk/solr/CHANGES.txt (props changed)
lucene/dev/trunk/solr/LICENSE.txt (props changed)
lucene/dev/trunk/solr/NOTICE.txt (props changed)
lucene/dev/trunk/solr/README.txt (props changed)
lucene/dev/trunk/solr/build.xml (props changed)
lucene/dev/trunk/solr/client/ (props changed)
lucene/dev/trunk/solr/common-build.xml (props changed)
lucene/dev/trunk/solr/contrib/ (props changed)
lucene/dev/trunk/solr/contrib/clustering/src/test-files/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler-extras/src/java/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test-files/ (props changed)
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/ (props changed)
lucene/dev/trunk/solr/contrib/uima/src/java/ (props changed)
lucene/dev/trunk/solr/contrib/uima/src/test-files/ (props changed)
lucene/dev/trunk/solr/core/ (props changed)
lucene/dev/trunk/solr/core/src/java/ (props changed)
lucene/dev/trunk/solr/core/src/test/ (props changed)
lucene/dev/trunk/solr/dev-tools/ (props changed)
lucene/dev/trunk/solr/example/ (props changed)
lucene/dev/trunk/solr/lib/ (props changed)
lucene/dev/trunk/solr/scripts/ (props changed)
lucene/dev/trunk/solr/site/ (props changed)
lucene/dev/trunk/solr/site-src/ (props changed)
lucene/dev/trunk/solr/solrj/ (props changed)
lucene/dev/trunk/solr/solrj/src/java/ (props changed)
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/ (props changed)
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/client/solrj/ (props changed)
lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/common/ (props changed)
lucene/dev/trunk/solr/test-framework/ (props changed)
lucene/dev/trunk/solr/testlogging.properties (props changed)
lucene/dev/trunk/solr/webapp/ (props changed)
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1190017&r1=1190016&r2=1190017&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Oct 27 20:49:13 2011
@@ -723,6 +723,9 @@ Optimizations
FilteredQuery/IndexSearcher added by LUCENE-1536 to Lucene 4.0.
(Uwe Schindler)
+* LUCENE-2205: Very substantial (3-5X) RAM reduction required to hold
+ the terms index on opening an IndexReader (Aaron McCurry via Mike McCandless)
+
Test Cases
* LUCENE-3420: Disable the finalness checks in TokenStream and Analyzer
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java?rev=1190017&r1=1190016&r2=1190017&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java Thu Oct 27 20:49:13 2011
@@ -47,9 +47,8 @@ public final class TermInfosReader {
private final SegmentTermEnum origEnum;
private final long size;
- private final Term[] indexTerms;
- private final TermInfo[] indexInfos;
- private final long[] indexPointers;
+ private final TermInfosReaderIndex index;
+ private final int indexLength;
private final int totalIndexInterval;
@@ -118,37 +117,23 @@ public final class TermInfosReader {
if (indexDivisor != -1) {
// Load terms index
totalIndexInterval = origEnum.indexInterval * indexDivisor;
- final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION),
- context), fieldInfos, true);
- try {
- int indexSize = 1+((int)indexEnum.size-1)/indexDivisor; // otherwise read index
+ final String indexFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION);
+ final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(indexFileName,
+ context), fieldInfos, true);
- indexTerms = new Term[indexSize];
- indexInfos = new TermInfo[indexSize];
- indexPointers = new long[indexSize];
-
- for (int i=0;indexEnum.next(); i++) {
- indexTerms[i] = indexEnum.term();
- assert indexTerms[i] != null;
- assert indexTerms[i].text() != null;
- assert indexTerms[i].field() != null;
- indexInfos[i] = indexEnum.termInfo();
- indexPointers[i] = indexEnum.indexPointer;
-
- for (int j = 1; j < indexDivisor; j++)
- if (!indexEnum.next())
- break;
- }
+ try {
+ // nocommit don't cast to int..
+ index = new TermInfosReaderIndex(indexEnum, indexDivisor, (int) dir.fileLength(indexFileName), totalIndexInterval);
+ indexLength = index.length();
} finally {
indexEnum.close();
}
} else {
// Do not load terms index:
totalIndexInterval = -1;
- indexTerms = null;
- indexInfos = null;
- indexPointers = null;
+ index = null;
+ indexLength = -1;
}
success = true;
} finally {
@@ -203,31 +188,6 @@ public final class TermInfosReader {
}
}
- /** Returns the offset of the greatest index entry which is less than or equal to term.*/
- private int getIndexOffset(Term term) {
- int lo = 0; // binary search indexTerms[]
- int hi = indexTerms.length - 1;
-
- while (hi >= lo) {
- int mid = (lo + hi) >>> 1;
- assert indexTerms[mid] != null : "indexTerms = " + indexTerms.length + " mid=" + mid;
- int delta = compareAsUTF16(term, indexTerms[mid]);
- if (delta < 0)
- hi = mid - 1;
- else if (delta > 0)
- lo = mid + 1;
- else
- return mid;
- }
- return hi;
- }
-
- private void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException {
- enumerator.seek(indexPointers[indexOffset],
- ((long) indexOffset * totalIndexInterval) - 1,
- indexTerms[indexOffset], indexInfos[indexOffset]);
- }
-
/** Returns the TermInfo for a Term in the set, or null. */
TermInfo get(Term term) throws IOException {
return get(term, false);
@@ -272,8 +232,8 @@ public final class TermInfosReader {
&& ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0)
|| compareAsUTF16(term, enumerator.term()) >= 0)) {
int enumOffset = (int)(enumerator.position/totalIndexInterval)+1;
- if (indexTerms.length == enumOffset // but before end of block
- || compareAsUTF16(term, indexTerms[enumOffset]) < 0) {
+ if (indexLength == enumOffset // but before end of block
+ || index.compareTo(term, enumOffset) < 0) {
// no need to seek
final TermInfo ti;
@@ -309,10 +269,10 @@ public final class TermInfosReader {
indexPos = (int) (tiOrd.termOrd / totalIndexInterval);
} else {
// Must do binary search:
- indexPos = getIndexOffset(term);
+ indexPos = index.getIndexOffset(term);
}
- seekEnum(enumerator, indexPos);
+ index.seekEnum(enumerator, indexPos);
enumerator.scanTo(term);
final TermInfo ti;
@@ -352,7 +312,7 @@ public final class TermInfosReader {
}
private void ensureIndexIsRead() {
- if (indexTerms == null) {
+ if (index == null) {
throw new IllegalStateException("terms index was not loaded when this reader was created");
}
}
@@ -362,10 +322,10 @@ public final class TermInfosReader {
if (size == 0) return -1;
ensureIndexIsRead();
- int indexOffset = getIndexOffset(term);
+ int indexOffset = index.getIndexOffset(term);
SegmentTermEnum enumerator = getThreadResources().termEnum;
- seekEnum(enumerator, indexOffset);
+ index.seekEnum(enumerator, indexOffset);
while(compareAsUTF16(term, enumerator.term()) > 0 && enumerator.next()) {}
Copied: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java (from r1189914, lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermInfosReaderIndex.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java?p2=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java&p1=lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermInfosReaderIndex.java&r1=1189914&r2=1190017&rev=1190017&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermInfosReaderIndex.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java Thu Oct 27 20:49:13 2011
@@ -1,4 +1,4 @@
-package org.apache.lucene.index;
+package org.apache.lucene.index.codecs.preflex;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
+import org.apache.lucene.index.Term;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
@@ -75,8 +76,8 @@ class TermInfosReaderIndex {
int fieldCounter = -1;
for (int i = 0; indexEnum.next(); i++) {
Term term = indexEnum.term();
- if (currentField != term.field) {
- currentField = term.field;
+ if (currentField == null || !currentField.equals(term.field())) {
+ currentField = term.field();
fieldStrs.add(currentField);
fieldCounter++;
}
@@ -120,7 +121,7 @@ class TermInfosReaderIndex {
// read the term
int fieldId = input.readVInt();
Term field = fields[fieldId];
- Term term = field.createTerm(input.readString());
+ Term term = new Term(field.field(), input.readString());
// read the terminfo
TermInfo termInfo = new TermInfo();
@@ -146,14 +147,14 @@ class TermInfosReaderIndex {
* the term to locate.
* @throws IOException
*/
- int getIndexOffset(Term term, BytesRef termBytesRef) throws IOException {
+ int getIndexOffset(Term term) throws IOException {
int lo = 0;
int hi = indexSize - 1;
PagedBytesDataInput input = (PagedBytesDataInput) dataInput.clone();
BytesRef scratch = new BytesRef();
while (hi >= lo) {
int mid = (lo + hi) >>> 1;
- int delta = compareTo(term, termBytesRef, mid, input, scratch);
+ int delta = compareTo(term, mid, input, scratch);
if (delta < 0)
hi = mid - 1;
else if (delta > 0)
@@ -179,7 +180,7 @@ class TermInfosReaderIndex {
// read the term
int fieldId = input.readVInt();
Term field = fields[fieldId];
- return field.createTerm(input.readString());
+ return new Term(field.field(), input.readString());
}
/**
@@ -202,8 +203,8 @@ class TermInfosReaderIndex {
* @return int.
* @throws IOException
*/
- int compareTo(Term term, BytesRef termBytesRef, int termIndex) throws IOException {
- return compareTo(term, termBytesRef, termIndex, (PagedBytesDataInput) dataInput.clone(), new BytesRef());
+ int compareTo(Term term, int termIndex) throws IOException {
+ return compareTo(term, termIndex, (PagedBytesDataInput) dataInput.clone(), new BytesRef());
}
/**
@@ -219,7 +220,7 @@ class TermInfosReaderIndex {
* @return int.
* @throws IOException
*/
- private int compareTo(Term term, BytesRef termBytesRef, int termIndex, PagedBytesDataInput input, BytesRef reuse) throws IOException {
+ private int compareTo(Term term, int termIndex, PagedBytesDataInput input, BytesRef reuse) throws IOException {
// if term field does not equal mid's field index, then compare fields
// else if they are equal, compare term's string values...
int c = compareField(term, termIndex, input);
@@ -227,7 +228,7 @@ class TermInfosReaderIndex {
reuse.length = input.readVInt();
reuse.grow(reuse.length);
input.readBytes(reuse.bytes, 0, reuse.length);
- return comparator.compare(termBytesRef, reuse);
+ return comparator.compare(term.bytes(), reuse);
}
return c;
}
@@ -246,6 +247,6 @@ class TermInfosReaderIndex {
*/
private int compareField(Term term, int termIndex, PagedBytesDataInput input) throws IOException {
input.setPosition(indexToDataOffset.get(termIndex));
- return term.field.compareTo(fields[input.readVInt()].field);
+ return term.field().compareTo(fields[input.readVInt()].field());
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java?rev=1190017&r1=1190016&r2=1190017&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java Thu Oct 27 20:49:13 2011
@@ -52,7 +52,7 @@ public final class ByteArrayDataInput ex
public int getPosition() {
return pos;
}
-
+
public void setPosition(int pos) {
this.pos = pos;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java?rev=1190017&r1=1190016&r2=1190017&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PagedBytes.java Thu Oct 27 20:49:13 2011
@@ -17,12 +17,14 @@ package org.apache.lucene.util;
* limitations under the License.
*/
-import org.apache.lucene.store.IndexInput;
-
-import java.util.List;
-import java.util.ArrayList;
import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexInput;
/** Represents a logical byte[] as a series of pages. You
* can write-once into the logical byte[] (append only),
@@ -37,6 +39,8 @@ public final class PagedBytes {
private final int blockSize;
private final int blockBits;
private final int blockMask;
+ private boolean didSkipBytes;
+ private boolean frozen;
private int upto;
private byte[] currentBlock;
@@ -320,6 +324,7 @@ public final class PagedBytes {
if (currentBlock != null) {
blocks.add(currentBlock);
blockEnd.add(upto);
+ didSkipBytes = true;
}
currentBlock = new byte[blockSize];
upto = 0;
@@ -338,6 +343,12 @@ public final class PagedBytes {
/** Commits final byte[], trimming it if necessary and if trim=true */
public Reader freeze(boolean trim) {
+ if (frozen) {
+ throw new IllegalStateException("already frozen");
+ }
+ if (didSkipBytes) {
+ throw new IllegalStateException("cannot freeze when copy(BytesRef, BytesRef) was used");
+ }
if (trim && upto < blockSize) {
final byte[] newBlock = new byte[upto];
System.arraycopy(currentBlock, 0, newBlock, 0, upto);
@@ -348,6 +359,7 @@ public final class PagedBytes {
}
blocks.add(currentBlock);
blockEnd.add(upto);
+ frozen = true;
currentBlock = null;
return new Reader(this);
}
@@ -389,4 +401,150 @@ public final class PagedBytes {
return pointer;
}
+
+ public final class PagedBytesDataInput extends DataInput {
+ private int currentBlockIndex;
+ private int currentBlockUpto;
+ private byte[] currentBlock;
+
+ PagedBytesDataInput() {
+ currentBlock = blocks.get(0);
+ }
+
+ @Override
+ public Object clone() {
+ PagedBytesDataInput clone = getDataInput();
+ clone.setPosition(getPosition());
+ return clone;
+ }
+
+ /** Returns the current byte position. */
+ public long getPosition() {
+ return currentBlockIndex * blockSize + currentBlockUpto;
+ }
+
+ /** Seek to a position previously obtained from
+ * {@link #getPosition}. */
+ public void setPosition(long pos) {
+ currentBlockIndex = (int) (pos >> blockBits);
+ currentBlock = blocks.get(currentBlockIndex);
+ currentBlockUpto = (int) (pos & blockMask);
+ }
+
+ @Override
+ public byte readByte() {
+ if (currentBlockUpto == blockSize) {
+ nextBlock();
+ }
+ return currentBlock[currentBlockUpto++];
+ }
+
+ @Override
+ public void readBytes(byte[] b, int offset, int len) {
+ final int offsetEnd = offset + len;
+ while (true) {
+ final int blockLeft = blockSize - currentBlockUpto;
+ final int left = offsetEnd - offset;
+ if (blockLeft < left) {
+ System.arraycopy(currentBlock, currentBlockUpto,
+ b, offset,
+ blockLeft);
+ nextBlock();
+ offset += blockLeft;
+ } else {
+ // Last block
+ System.arraycopy(currentBlock, currentBlockUpto,
+ b, offset,
+ left);
+ currentBlockUpto += left;
+ break;
+ }
+ }
+ }
+
+ private void nextBlock() {
+ currentBlockIndex++;
+ currentBlockUpto = 0;
+ currentBlock = blocks.get(currentBlockIndex);
+ }
+ }
+
+ public final class PagedBytesDataOutput extends DataOutput {
+ @Override
+ public void writeByte(byte b) {
+ if (upto == blockSize) {
+ if (currentBlock != null) {
+ blocks.add(currentBlock);
+ blockEnd.add(upto);
+ }
+ currentBlock = new byte[blockSize];
+ upto = 0;
+ }
+ currentBlock[upto++] = b;
+ }
+
+ @Override
+ public void writeBytes(byte[] b, int offset, int length) throws IOException {
+ if (length == 0) {
+ return;
+ }
+
+ if (upto == blockSize) {
+ if (currentBlock != null) {
+ blocks.add(currentBlock);
+ blockEnd.add(upto);
+ }
+ currentBlock = new byte[blockSize];
+ upto = 0;
+ }
+
+ final int offsetEnd = offset + length;
+ while(true) {
+ final int left = offsetEnd - offset;
+ final int blockLeft = blockSize - upto;
+ if (blockLeft < left) {
+ System.arraycopy(b, offset, currentBlock, upto, blockLeft);
+ blocks.add(currentBlock);
+ blockEnd.add(blockSize);
+ currentBlock = new byte[blockSize];
+ upto = 0;
+ offset += blockLeft;
+ } else {
+ // Last block
+ System.arraycopy(b, offset, currentBlock, upto, left);
+ upto += left;
+ break;
+ }
+ }
+ }
+
+ /** Return the current byte position. */
+ public long getPosition() {
+ if (currentBlock == null) {
+ return 0;
+ } else {
+ return blocks.size() * blockSize + upto;
+ }
+ }
+ }
+
+ /** Returns a DataInput to read values from this
+ * PagedBytes instance. */
+ public PagedBytesDataInput getDataInput() {
+ if (!frozen) {
+ throw new IllegalStateException("must call freeze() before getDataInput");
+ }
+ return new PagedBytesDataInput();
+ }
+
+ /** Returns a DataOutput that you may use to write into
+ * this PagedBytes instance. If you do this, you should
+ * not call the other writing methods (eg, copy);
+ * results are undefined. */
+ public PagedBytesDataOutput getDataOutput() {
+ if (frozen) {
+ throw new IllegalStateException("cannot get DataOutput after freeze()");
+ }
+ return new PagedBytesDataOutput();
+ }
}
Copied: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java (from r1189914, lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermInfosReaderIndex.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java?p2=lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java&p1=lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermInfosReaderIndex.java&r1=1189914&r2=1190017&rev=1190017&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermInfosReaderIndex.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java Thu Oct 27 20:49:13 2011
@@ -1,4 +1,4 @@
-package org.apache.lucene.index;
+package org.apache.lucene.index.codecs.preflex;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -26,8 +26,23 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.CoreCodecProvider;
+import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
@@ -47,7 +62,6 @@ public class TestTermInfosReaderIndex ex
private SegmentTermEnum termEnum;
private int indexDivisor;
private int termIndexInterval;
- private int readBufferSize = 1024;
private IndexReader reader;
private List<Term> sampleTerms;
@@ -57,16 +71,17 @@ public class TestTermInfosReaderIndex ex
indexDivisor = _TestUtil.nextInt(random, 1, 10);
directory = newDirectory();
termIndexInterval = populate(directory);
-
- SegmentReader r = SegmentReader.getOnlySegmentReader(directory);
+
+ IndexReader r0 = IndexReader.open(directory);
+ SegmentReader r = (SegmentReader) r0.getSequentialSubReaders()[0];
String segment = r.getSegmentName();
r.close();
- FieldInfos fieldInfos = new FieldInfos(directory, IndexFileNames.segmentFileName(segment, IndexFileNames.FIELD_INFOS_EXTENSION));
- String segmentFileName = IndexFileNames.segmentFileName(segment, IndexFileNames.TERMS_INDEX_EXTENSION);
+ FieldInfos fieldInfos = new FieldInfos(directory, IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELD_INFOS_EXTENSION));
+ String segmentFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION);
long tiiFileLength = directory.fileLength(segmentFileName);
- IndexInput input = directory.openInput(segmentFileName, readBufferSize);
- termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.TERMS_EXTENSION), readBufferSize), fieldInfos, false);
+ IndexInput input = directory.openInput(segmentFileName, newIOContext(random));
+ termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), newIOContext(random)), fieldInfos, false);
int totalIndexInterval = termEnum.indexInterval * indexDivisor;
SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true);
@@ -91,19 +106,18 @@ public class TestTermInfosReaderIndex ex
int indexPosition = 3;
SegmentTermEnum clone = (SegmentTermEnum) termEnum.clone();
Term term = findTermThatWouldBeAtIndex(clone, indexPosition);
- clone.close();
SegmentTermEnum enumerator = clone;
index.seekEnum(enumerator, indexPosition);
assertEquals(term, enumerator.term());
+ clone.close();
}
public void testCompareTo() throws IOException {
Term term = new Term("field" + random.nextInt(NUMBER_OF_FIELDS) ,getText());
- BytesRef termBytesRef = new BytesRef(term.text);
for (int i = 0; i < index.length(); i++) {
Term t = index.getTerm(i);
int compareTo = term.compareTo(t);
- assertEquals(compareTo, index.compareTo(term, termBytesRef, i));
+ assertEquals(compareTo, index.compareTo(term, i));
}
}
@@ -120,16 +134,19 @@ public class TestTermInfosReaderIndex ex
private List<Term> sample(IndexReader reader, int size) throws IOException {
List<Term> sample = new ArrayList<Term>();
Random random = new Random();
- TermEnum terms = reader.terms();
- while (terms.next()) {
- if (sample.size() >= size) {
- int pos = random.nextInt(size);
- sample.set(pos, terms.term());
- } else {
- sample.add(terms.term());
+ FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator();
+ String field;
+ while((field = fieldsEnum.next()) != null) {
+ TermsEnum terms = fieldsEnum.terms();
+ while (terms.next() != null) {
+ if (sample.size() >= size) {
+ int pos = random.nextInt(size);
+ sample.set(pos, new Term(field, terms.term()));
+ } else {
+ sample.add(new Term(field, terms.term()));
+ }
}
}
- terms.close();
Collections.shuffle(sample);
return sample;
}
@@ -147,6 +164,11 @@ public class TestTermInfosReaderIndex ex
private int populate(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException {
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random, MockTokenizer.KEYWORD, false));
+ CoreCodecProvider cp = new CoreCodecProvider();
+ cp.unregister(cp.lookup("PreFlex"));
+ cp.register(new PreFlexRWCodec());
+ cp.setDefaultFieldCodec("PreFlex");
+ config.setCodecProvider(cp);
// turn off compound file, this test will open some index files directly.
LogMergePolicy mp = newLogMergePolicy();
mp.setUseCompoundFile(false);
@@ -156,7 +178,7 @@ public class TestTermInfosReaderIndex ex
for (int i = 0; i < NUMBER_OF_DOCUMENTS; i++) {
Document document = new Document();
for (int f = 0; f < NUMBER_OF_FIELDS; f++) {
- document.add(newField("field" + f,getText(),Store.NO,Index.NOT_ANALYZED_NO_NORMS));
+ document.add(newField("field" + f, getText(), StringField.TYPE_UNSTORED));
}
writer.addDocument(document);
}