You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/07/21 11:24:02 UTC
svn commit: r1612213 - in /lucene/dev/branches/branch_4x: ./
dev-tools/scripts/ lucene/ lucene/codecs/
lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/
lucene/codecs/src/resources/META-INF/services/
lucene/codecs/src/test/org/apache/lucen...
Author: mikemccand
Date: Mon Jul 21 09:24:01 2014
New Revision: 1612213
URL: http://svn.apache.org/r1612213
Log:
LUCENE-5819: add terms dict and postings format that implement term ordinals
Added:
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/
- copied from r1612080, lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/
lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/
- copied from r1612080, lucene/dev/trunk/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/scripts/diffSources.py
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnumFrame.java
lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnumFrame.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
Modified: lucene/dev/branches/branch_4x/dev-tools/scripts/diffSources.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/dev-tools/scripts/diffSources.py?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/dev-tools/scripts/diffSources.py (original)
+++ lucene/dev/branches/branch_4x/dev-tools/scripts/diffSources.py Mon Jul 21 09:24:01 2014
@@ -60,7 +60,7 @@ while True:
keep = True
else:
ext = os.path.splitext(l)[-1]
- keep = ext in ('.xml', '.iml', '.html', '.template', '.py', '.g', '.properties')
+ keep = ext in ('.xml', '.iml', '.html', '.template', '.py', '.g', '.properties', '.java')
if keep:
print
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Mon Jul 21 09:24:01 2014
@@ -31,6 +31,11 @@ New Features
"any" term) are allowed. This is a generalization of
MultiPhraseQuery and span queries, and enables "correct" (including
position) length search-time graph synonyms. (Mike McCandless)
+
+* LUCENE-5819: Add OrdsLucene41 block tree terms dict and postings
+ format, to include term ordinals in the index so the optional
+ TermsEnum.ord() and TermsEnum.seekExact(long ord) APIs work. (Mike
+ McCandless)
API Changes
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java?rev=1612213&r1=1612080&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java Mon Jul 21 09:24:01 2014
@@ -19,12 +19,16 @@ package org.apache.lucene.codecs.blocktr
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.codecs.PostingsConsumer;
import org.apache.lucene.codecs.PostingsWriterBase;
+import org.apache.lucene.codecs.TermStats;
+import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter;
import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -135,6 +139,7 @@ public final class OrdsBlockTreeTermsWri
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
+ FieldInfo currentField;
private static class FieldMetaData {
public final FieldInfo fieldInfo;
@@ -225,40 +230,14 @@ public final class OrdsBlockTreeTermsWri
this.indexOut = indexOut;
}
- @Override
- public void write(Fields fields) throws IOException {
-
- String lastField = null;
- for(String field : fields) {
- assert lastField == null || lastField.compareTo(field) < 0;
- lastField = field;
-
- Terms terms = fields.terms(field);
- if (terms == null) {
- continue;
- }
-
- TermsEnum termsEnum = terms.iterator(null);
-
- TermsWriter termsWriter = new TermsWriter(fieldInfos.fieldInfo(field));
- BytesRef minTerm = null;
- BytesRef maxTerm = new BytesRef();
- while (true) {
- BytesRef term = termsEnum.next();
- if (term == null) {
- break;
- }
- if (minTerm == null) {
- minTerm = BytesRef.deepCopyOf(term);
- }
- maxTerm.copyBytes(term);
- termsWriter.write(term, termsEnum);
- }
-
- termsWriter.finish(minTerm, minTerm == null ? null : maxTerm);
- }
+ public TermsConsumer addField(FieldInfo field) throws IOException {
+ //DEBUG = field.name.equals("id");
+ //if (DEBUG) System.out.println("\nBTTW.addField seg=" + segment + " field=" + field.name);
+ assert currentField == null || currentField.name.compareTo(field.name) < 0;
+ currentField = field;
+ return new TermsWriter(field);
}
-
+
static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {
assert fp < (1L << 62);
return (fp << 2) | (hasTerms ? OUTPUT_FLAG_HAS_TERMS : 0) | (isFloor ? OUTPUT_FLAG_IS_FLOOR : 0);
@@ -432,13 +411,13 @@ public final class OrdsBlockTreeTermsWri
final RAMOutputStream scratchBytes = new RAMOutputStream();
- class TermsWriter {
+ class TermsWriter extends TermsConsumer {
private final FieldInfo fieldInfo;
private final int longsSize;
private long numTerms;
- final FixedBitSet docsSeen;
long sumTotalTermFreq;
long sumDocFreq;
+ int docCount;
long indexStartFP;
// Used only to partition terms into the block tree; we
@@ -459,6 +438,9 @@ public final class OrdsBlockTreeTermsWri
private int[] subTermCountSums = new int[10];
private int[] subSubCounts = new int[10];
+ private BytesRef minTerm;
+ private BytesRef maxTerm = new BytesRef();
+
// This class assigns terms to blocks "naturally", ie,
// according to the number of terms under a given prefix
// that we encounter:
@@ -972,7 +954,6 @@ public final class OrdsBlockTreeTermsWri
TermsWriter(FieldInfo fieldInfo) {
this.fieldInfo = fieldInfo;
- docsSeen = new FixedBitSet(maxDoc);
noOutputs = NoOutputs.getSingleton();
@@ -990,26 +971,50 @@ public final class OrdsBlockTreeTermsWri
this.longsSize = postingsWriter.setField(fieldInfo);
}
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
+ public PostingsConsumer startTerm(BytesRef text) throws IOException {
+ //if (DEBUG) System.out.println("\nBTTW.startTerm term=" + fieldInfo.name + ":" + toString(text) + " seg=" + segment);
+ postingsWriter.startTerm();
+ /*
+ if (fieldInfo.name.equals("id")) {
+ postingsWriter.termID = Integer.parseInt(text.utf8ToString());
+ } else {
+ postingsWriter.termID = -1;
+ }
+ */
+ return postingsWriter;
+ }
+
private final IntsRef scratchIntsRef = new IntsRef();
/** Writes one term's worth of postings. */
- public void write(BytesRef text, TermsEnum termsEnum) throws IOException {
- BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen);
- if (state != null) {
- assert state.docFreq != 0;
- assert fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter;
- sumDocFreq += state.docFreq;
- sumTotalTermFreq += state.totalTermFreq;
- blockBuilder.add(Util.toIntsRef(text, scratchIntsRef), noOutputs.getNoOutput());
-
- PendingTerm term = new PendingTerm(BytesRef.deepCopyOf(text), state);
- pending.add(term);
- numTerms++;
+ @Override
+ public void finishTerm(BytesRef text, TermStats stats) throws IOException {
+ assert stats.docFreq != 0;
+
+ blockBuilder.add(Util.toIntsRef(text, scratchIntsRef), noOutputs.getNoOutput());
+ BlockTermState state = postingsWriter.newTermState();
+ state.docFreq = stats.docFreq;
+ state.totalTermFreq = stats.totalTermFreq;
+ postingsWriter.finishTerm(state);
+
+ PendingTerm term = new PendingTerm(BytesRef.deepCopyOf(text), state);
+ pending.add(term);
+ numTerms++;
+
+ if (minTerm == null) {
+ minTerm = BytesRef.deepCopyOf(text);
}
+ maxTerm.copyBytes(text);
}
// Finishes all terms in this field
- public void finish(BytesRef minTerm, BytesRef maxTerm) throws IOException {
+ public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException {
if (numTerms > 0) {
blockBuilder.finish();
@@ -1019,6 +1024,10 @@ public final class OrdsBlockTreeTermsWri
assert root.prefix.length == 0;
assert root.index.getEmptyOutput() != null;
+ this.sumTotalTermFreq = sumTotalTermFreq;
+ this.sumDocFreq = sumDocFreq;
+ this.docCount = docCount;
+
// Write FST to index
indexStartFP = indexOut.getFilePointer();
root.index.save(indexOut);
@@ -1038,11 +1047,13 @@ public final class OrdsBlockTreeTermsWri
indexStartFP,
sumTotalTermFreq,
sumDocFreq,
- docsSeen.cardinality(),
+ docCount,
longsSize,
minTerm, maxTerm));
} else {
- assert docsSeen.cardinality() == 0;
+ assert sumTotalTermFreq == 0 || fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY && sumTotalTermFreq == -1;
+ assert sumDocFreq == 0;
+ assert docCount == 0;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java?rev=1612213&r1=1612080&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsFieldReader.java Mon Jul 21 09:24:01 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.blocktr
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -114,6 +115,11 @@ final class OrdsFieldReader extends Term
}
@Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
public boolean hasFreqs() {
return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java?rev=1612213&r1=1612080&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java Mon Jul 21 09:24:01 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.blocktr
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -464,6 +465,11 @@ final class OrdsIntersectTermsEnum exten
}
@Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
public boolean seekExact(BytesRef text) {
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java?rev=1612213&r1=1612080&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java Mon Jul 21 09:24:01 2014
@@ -22,6 +22,7 @@ package org.apache.lucene.codecs.blocktr
import java.io.IOException;
import java.io.PrintStream;
+import java.util.Comparator;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
@@ -147,6 +148,11 @@ public final class OrdsSegmentTermsEnum
return arcs[ord];
}
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
// Pushes a frame we seek'd to
OrdsSegmentTermsEnumFrame pushFrame(FST.Arc<Output> arc, Output frameData, int length) throws IOException {
scratchReader.reset(frameData.bytes.bytes, frameData.bytes.offset, frameData.bytes.length);
@@ -186,6 +192,7 @@ public final class OrdsSegmentTermsEnum
// }
}
assert length == f.prefix;
+ assert termOrd == f.termOrdOrig;
} else {
f.nextEnt = -1;
f.prefix = length;
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnumFrame.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnumFrame.java?rev=1612213&r1=1612080&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnumFrame.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnumFrame.java Mon Jul 21 09:24:01 2014
@@ -655,19 +655,6 @@ final class OrdsSegmentTermsEnumFrame {
// return NOT_FOUND:
fillTerm();
- if (!exactOnly && !ste.termExists) {
- // We are on a sub-block, and caller wants
- // us to position to the next term after
- // the target, so we must recurse into the
- // sub-frame(s):
- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen, ste.currentFrame.termOrd);
- ste.currentFrame.loadBlock();
- while (ste.currentFrame.next()) {
- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length, ste.currentFrame.termOrd);
- ste.currentFrame.loadBlock();
- }
- }
-
//if (DEBUG) System.out.println(" not found");
return SeekStatus.NOT_FOUND;
} else if (stop) {
@@ -742,6 +729,8 @@ final class OrdsSegmentTermsEnumFrame {
final int termLen = prefix + suffix;
startBytePos = suffixesReader.getPosition();
suffixesReader.skipBytes(suffix);
+ // Must save ord before we skip over a sub-block in case we push, below:
+ long prevTermOrd = termOrd;
if (ste.termExists) {
state.termBlockOrd++;
termOrd++;
@@ -795,10 +784,10 @@ final class OrdsSegmentTermsEnumFrame {
// us to position to the next term after
// the target, so we must recurse into the
// sub-frame(s):
- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen, ste.currentFrame.termOrd);
+ ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen, prevTermOrd);
ste.currentFrame.loadBlock();
while (ste.currentFrame.next()) {
- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length, ste.currentFrame.termOrd);
+ ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length, prevTermOrd);
ste.currentFrame.loadBlock();
}
}
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Mon Jul 21 09:24:01 2014
@@ -13,12 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat
-org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
-org.apache.lucene.codecs.memory.MemoryPostingsFormat
+org.apache.lucene.codecs.blocktreeords.Ords41PostingsFormat
org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat
org.apache.lucene.codecs.memory.DirectPostingsFormat
-org.apache.lucene.codecs.memory.FSTPulsing41PostingsFormat
+org.apache.lucene.codecs.memory.FSTOrdPostingsFormat
org.apache.lucene.codecs.memory.FSTOrdPulsing41PostingsFormat
org.apache.lucene.codecs.memory.FSTPostingsFormat
-org.apache.lucene.codecs.memory.FSTOrdPostingsFormat
+org.apache.lucene.codecs.memory.FSTPulsing41PostingsFormat
+org.apache.lucene.codecs.memory.MemoryPostingsFormat
+org.apache.lucene.codecs.pulsing.Pulsing41PostingsFormat
+org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java?rev=1612213&r1=1612080&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/test/org/apache/lucene/codecs/blocktreeords/TestOrdsBlockTree.java Mon Jul 21 09:24:01 2014
@@ -151,6 +151,33 @@ public class TestOrdsBlockTree extends B
dir.close();
}
+ public void testSeekCeilNotFound() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ Document doc = new Document();
+ // Get empty string in there!
+ doc.add(newStringField("field", "", Field.Store.NO));
+ w.addDocument(doc);
+
+ for(int i=0;i<36;i++) {
+ doc = new Document();
+ String term = "" + (char) (97+i);
+ String term2 = "a" + (char) (97+i);
+ doc.add(newTextField("field", term + " " + term2, Field.Store.NO));
+ w.addDocument(doc);
+ }
+
+ w.forceMerge(1);
+ IndexReader r = w.getReader();
+ TermsEnum te = MultiFields.getTerms(r, "field").iterator(null);
+ assertEquals(TermsEnum.SeekStatus.NOT_FOUND, te.seekCeil(new BytesRef(new byte[] {0x22})));
+ assertEquals("a", te.term().utf8ToString());
+ assertEquals(1L, te.ord());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
public void testThreeBlocks() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnumFrame.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnumFrame.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnumFrame.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnumFrame.java Mon Jul 21 09:24:01 2014
@@ -222,6 +222,7 @@ final class SegmentTermsEnumFrame {
if (isFloor) {
floorDataReader.rewind();
numFollowFloorBlocks = floorDataReader.readVInt();
+ assert numFollowFloorBlocks > 0;
nextFloorLabel = floorDataReader.readByte() & 0xff;
}
@@ -551,19 +552,6 @@ final class SegmentTermsEnumFrame {
// return NOT_FOUND:
fillTerm();
- if (!exactOnly && !ste.termExists) {
- // We are on a sub-block, and caller wants
- // us to position to the next term after
- // the target, so we must recurse into the
- // sub-frame(s):
- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen);
- ste.currentFrame.loadBlock();
- while (ste.currentFrame.next()) {
- ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length);
- ste.currentFrame.loadBlock();
- }
- }
-
//if (DEBUG) System.out.println(" not found");
return SeekStatus.NOT_FOUND;
} else if (stop) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Mon Jul 21 09:24:01 2014
@@ -45,6 +45,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LongBitSet;
import org.apache.lucene.util.StringHelper;
@@ -344,6 +345,20 @@ public class CheckIndex {
return crossCheckTermVectors;
}
+ private boolean failFast;
+
+ /** If true, just throw the original exception immediately when
+ * corruption is detected, rather than continuing to iterate to other
+ * segments looking for more corruption. */
+ public void setFailFast(boolean v) {
+ failFast = v;
+ }
+
+ /** See {@link #setFailFast}. */
+ public boolean getFailFast() {
+ return failFast;
+ }
+
private boolean verbose;
/** Set infoStream where messages should go. If null, no
@@ -397,6 +412,9 @@ public class CheckIndex {
try {
sis.read(dir);
} catch (Throwable t) {
+ if (failFast) {
+ IOUtils.reThrow(t);
+ }
msg(infoStream, "ERROR: could not read any segments file in directory");
result.missingSegments = true;
if (infoStream != null)
@@ -432,6 +450,9 @@ public class CheckIndex {
try {
input = dir.openInput(segmentsFileName, IOContext.READONCE);
} catch (Throwable t) {
+ if (failFast) {
+ IOUtils.reThrow(t);
+ }
msg(infoStream, "ERROR: could not open segments file in directory");
if (infoStream != null)
t.printStackTrace(infoStream);
@@ -442,6 +463,9 @@ public class CheckIndex {
try {
format = input.readInt();
} catch (Throwable t) {
+ if (failFast) {
+ IOUtils.reThrow(t);
+ }
msg(infoStream, "ERROR: could not read segment file version in directory");
if (infoStream != null)
t.printStackTrace(infoStream);
@@ -625,18 +649,18 @@ public class CheckIndex {
segInfoStat.numFields = fieldInfos.size();
// Test Field Norms
- segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream);
+ segInfoStat.fieldNormStatus = testFieldNorms(reader, infoStream, failFast);
// Test the Term Index
- segInfoStat.termIndexStatus = testPostings(reader, infoStream, verbose);
+ segInfoStat.termIndexStatus = testPostings(reader, infoStream, verbose, failFast);
// Test Stored Fields
- segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream);
+ segInfoStat.storedFieldStatus = testStoredFields(reader, infoStream, failFast);
// Test Term Vectors
- segInfoStat.termVectorStatus = testTermVectors(reader, infoStream, verbose, crossCheckTermVectors);
+ segInfoStat.termVectorStatus = testTermVectors(reader, infoStream, verbose, crossCheckTermVectors, failFast);
- segInfoStat.docValuesStatus = testDocValues(reader, infoStream);
+ segInfoStat.docValuesStatus = testDocValues(reader, infoStream, failFast);
// Rethrow the first exception we encountered
// This will cause stats for failed segments to be incremented properly
@@ -655,6 +679,9 @@ public class CheckIndex {
msg(infoStream, "");
} catch (Throwable t) {
+ if (failFast) {
+ IOUtils.reThrow(t);
+ }
msg(infoStream, "FAILED");
String comment;
comment = "fixIndex() would remove reference to this segment";
@@ -696,7 +723,7 @@ public class CheckIndex {
* Test field norms.
* @lucene.experimental
*/
- public static Status.FieldNormStatus testFieldNorms(AtomicReader reader, PrintStream infoStream) {
+ public static Status.FieldNormStatus testFieldNorms(AtomicReader reader, PrintStream infoStream, boolean failFast) throws IOException {
final Status.FieldNormStatus status = new Status.FieldNormStatus();
try {
@@ -719,6 +746,9 @@ public class CheckIndex {
msg(infoStream, "OK [" + status.totFields + " fields]");
} catch (Throwable e) {
+ if (failFast) {
+ IOUtils.reThrow(e);
+ }
msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
status.error = e;
if (infoStream != null) {
@@ -847,6 +877,7 @@ public class CheckIndex {
long sumTotalTermFreq = 0;
long sumDocFreq = 0;
+ long upto = 0;
FixedBitSet visitedDocs = new FixedBitSet(maxDoc);
while(true) {
@@ -854,7 +885,7 @@ public class CheckIndex {
if (term == null) {
break;
}
-
+
assert term.isValid();
// make sure terms arrive in order according to
@@ -1298,15 +1329,15 @@ public class CheckIndex {
* Test the term index.
* @lucene.experimental
*/
- public static Status.TermIndexStatus testPostings(AtomicReader reader, PrintStream infoStream) {
- return testPostings(reader, infoStream, false);
+ public static Status.TermIndexStatus testPostings(AtomicReader reader, PrintStream infoStream) throws IOException {
+ return testPostings(reader, infoStream, false, false);
}
/**
* Test the term index.
* @lucene.experimental
*/
- public static Status.TermIndexStatus testPostings(AtomicReader reader, PrintStream infoStream, boolean verbose) {
+ public static Status.TermIndexStatus testPostings(AtomicReader reader, PrintStream infoStream, boolean verbose, boolean failFast) throws IOException {
// TODO: we should go and verify term vectors match, if
// crossCheckTermVectors is on...
@@ -1330,6 +1361,9 @@ public class CheckIndex {
checkFields(fields, null, maxDoc, fieldInfos, true, false, infoStream, verbose);
}
} catch (Throwable e) {
+ if (failFast) {
+ IOUtils.reThrow(e);
+ }
msg(infoStream, "ERROR: " + e);
status = new Status.TermIndexStatus();
status.error = e;
@@ -1345,7 +1379,7 @@ public class CheckIndex {
* Test stored fields.
* @lucene.experimental
*/
- public static Status.StoredFieldStatus testStoredFields(AtomicReader reader, PrintStream infoStream) {
+ public static Status.StoredFieldStatus testStoredFields(AtomicReader reader, PrintStream infoStream, boolean failFast) throws IOException {
final Status.StoredFieldStatus status = new Status.StoredFieldStatus();
try {
@@ -1373,6 +1407,9 @@ public class CheckIndex {
msg(infoStream, "OK [" + status.totFields + " total field count; avg " +
NumberFormat.getInstance(Locale.ROOT).format((((float) status.totFields)/status.docCount)) + " fields per doc]");
} catch (Throwable e) {
+ if (failFast) {
+ IOUtils.reThrow(e);
+ }
msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
status.error = e;
if (infoStream != null) {
@@ -1388,7 +1425,8 @@ public class CheckIndex {
* @lucene.experimental
*/
public static Status.DocValuesStatus testDocValues(AtomicReader reader,
- PrintStream infoStream) {
+ PrintStream infoStream,
+ boolean failFast) throws IOException {
final Status.DocValuesStatus status = new Status.DocValuesStatus();
try {
if (infoStream != null) {
@@ -1416,6 +1454,9 @@ public class CheckIndex {
+ status.totalSortedNumericFields + " SORTED_NUMERIC; "
+ status.totalSortedSetFields + " SORTED_SET]");
} catch (Throwable e) {
+ if (failFast) {
+ IOUtils.reThrow(e);
+ }
msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
status.error = e;
if (infoStream != null) {
@@ -1655,15 +1696,15 @@ public class CheckIndex {
* Test term vectors.
* @lucene.experimental
*/
- public static Status.TermVectorStatus testTermVectors(AtomicReader reader, PrintStream infoStream) {
- return testTermVectors(reader, infoStream, false, false);
+ public static Status.TermVectorStatus testTermVectors(AtomicReader reader, PrintStream infoStream) throws IOException {
+ return testTermVectors(reader, infoStream, false, false, false);
}
/**
* Test term vectors.
* @lucene.experimental
*/
- public static Status.TermVectorStatus testTermVectors(AtomicReader reader, PrintStream infoStream, boolean verbose, boolean crossCheckTermVectors) {
+ public static Status.TermVectorStatus testTermVectors(AtomicReader reader, PrintStream infoStream, boolean verbose, boolean crossCheckTermVectors, boolean failFast) throws IOException {
final Status.TermVectorStatus status = new Status.TermVectorStatus();
final FieldInfos fieldInfos = reader.getFieldInfos();
final Bits onlyDocIsDeleted = new FixedBitSet(1);
@@ -1875,6 +1916,9 @@ public class CheckIndex {
msg(infoStream, "OK [" + status.totVectors + " total vector count; avg " +
NumberFormat.getInstance(Locale.ROOT).format(vectorAvg) + " term/freq vector fields per doc]");
} catch (Throwable e) {
+ if (failFast) {
+ IOUtils.reThrow(e);
+ }
msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
status.error = e;
if (infoStream != null) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Mon Jul 21 09:24:01 2014
@@ -23,6 +23,7 @@ import org.apache.lucene.store.DataInput
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.StringHelper;
/**
* An FST {@link Outputs} implementation where each output
@@ -80,13 +81,16 @@ public final class ByteSequenceOutputs e
if (inc == NO_OUTPUT) {
// no prefix removed
return output;
- } else if (inc.length == output.length) {
- // entire output removed
- return NO_OUTPUT;
} else {
- assert inc.length < output.length: "inc.length=" + inc.length + " vs output.length=" + output.length;
- assert inc.length > 0;
- return new BytesRef(output.bytes, output.offset + inc.length, output.length-inc.length);
+ assert StringHelper.startsWith(output, inc);
+ if (inc.length == output.length) {
+ // entire output removed
+ return NO_OUTPUT;
+ } else {
+ assert inc.length < output.length: "inc.length=" + inc.length + " vs output.length=" + output.length;
+ assert inc.length > 0;
+ return new BytesRef(output.bytes, output.offset + inc.length, output.length-inc.length);
+ }
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Mon Jul 21 09:24:01 2014
@@ -85,7 +85,10 @@ public final class FST<T> implements Acc
// TODO: we can free up a bit if we can nuke this:
final static int BIT_STOP_NODE = 1 << 3;
- final static int BIT_ARC_HAS_OUTPUT = 1 << 4;
+
+ /** This flag is set if the arc has an output. */
+ public final static int BIT_ARC_HAS_OUTPUT = 1 << 4;
+
final static int BIT_ARC_HAS_FINAL_OUTPUT = 1 << 5;
// Arcs are stored as fixed-size (per entry) array, so
@@ -196,11 +199,22 @@ public final class FST<T> implements Acc
// address (into the byte[]), or ord/address if label == END_LABEL
long nextArc;
- // This is non-zero if current arcs are fixed array:
- long posArcsStart;
- int bytesPerArc;
- int arcIdx;
- int numArcs;
+ /** Where the first arc in the array starts; only valid if
+ * bytesPerArc != 0 */
+ public long posArcsStart;
+
+ /** Non-zero if this arc is part of an array, which means all
+ * arcs for the node are encoded with a fixed number of bytes so
+ * that we can random access by index. We do when there are enough
+ * arcs leaving one node. It wastes some bytes but gives faster
+ * lookups. */
+ public int bytesPerArc;
+
+ /** Where we are in the array; only valid if bytesPerArc != 0. */
+ public int arcIdx;
+
+ /** How many arcs in the array; only valid if bytesPerArc != 0. */
+ public int numArcs;
/** Returns this */
public Arc<T> copyFrom(Arc<T> other) {
@@ -644,7 +658,8 @@ public final class FST<T> implements Acc
}
}
- int readLabel(DataInput in) throws IOException {
+ /** Reads one BYTE1/2/4 label from the provided {@link DataInput}. */
+ public int readLabel(DataInput in) throws IOException {
final int v;
if (inputType == INPUT_TYPE.BYTE1) {
// Unsigned byte:
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Mon Jul 21 09:24:01 2014
@@ -1029,6 +1029,7 @@ public class TestIndexWriterReader exten
(format.equals("FSTPulsing41") ||
format.equals("FSTOrdPulsing41") ||
format.equals("FST41") ||
+ format.equals("OrdsLucene41") ||
format.equals("FSTOrd41") ||
format.equals("SimpleText") ||
format.equals("Memory") ||
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Mon Jul 21 09:24:01 2014
@@ -38,6 +38,8 @@ import org.apache.lucene.codecs.blockter
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader;
import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter;
+import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader;
+import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsWriter;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
import org.apache.lucene.codecs.mockintblock.MockFixedIntBlockPostingsFormat;
@@ -191,7 +193,7 @@ public final class MockRandomPostingsFor
}
final FieldsConsumer fields;
- final int t1 = random.nextInt(4);
+ final int t1 = random.nextInt(5);
if (t1 == 0) {
boolean success = false;
@@ -234,7 +236,7 @@ public final class MockRandomPostingsFor
postingsWriter.close();
}
}
- } else {
+ } else if (t1 == 3) {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing Block terms dict");
@@ -304,6 +306,30 @@ public final class MockRandomPostingsFor
}
}
}
+ } else if (t1 == 4) {
+ // Use OrdsBlockTree terms dict
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: writing OrdsBlockTree");
+ }
+
+ // TODO: would be nice to allow 1 but this is very
+ // slow to write
+ final int minTermsInBlock = TestUtil.nextInt(random, 2, 100);
+ final int maxTermsInBlock = Math.max(2, (minTermsInBlock-1)*2 + random.nextInt(100));
+
+ boolean success = false;
+ try {
+ fields = new OrdsBlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock);
+ success = true;
+ } finally {
+ if (!success) {
+ postingsWriter.close();
+ }
+ }
+
+ } else {
+ // BUG!
+ throw new AssertionError();
}
return fields;
@@ -351,7 +377,7 @@ public final class MockRandomPostingsFor
}
final FieldsProducer fields;
- final int t1 = random.nextInt(4);
+ final int t1 = random.nextInt(5);
if (t1 == 0) {
boolean success = false;
try {
@@ -393,7 +419,7 @@ public final class MockRandomPostingsFor
postingsReader.close();
}
}
- } else {
+ } else if (t1 == 3) {
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading Block terms dict");
@@ -464,6 +490,29 @@ public final class MockRandomPostingsFor
}
}
}
+ } else if (t1 == 4) {
+ // Use OrdsBlockTree terms dict
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MockRandomCodec: reading OrdsBlockTree terms dict");
+ }
+
+ boolean success = false;
+ try {
+ fields = new OrdsBlockTreeTermsReader(state.directory,
+ state.fieldInfos,
+ state.segmentInfo,
+ postingsReader,
+ state.context,
+ state.segmentSuffix);
+ success = true;
+ } finally {
+ if (!success) {
+ postingsReader.close();
+ }
+ }
+ } else {
+ // BUG!
+ throw new AssertionError();
}
return fields;
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Mon Jul 21 09:24:01 2014
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.Set;
+import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.lucene.codecs.Codec;
@@ -51,6 +52,11 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.AutomatonTestUtil.RandomAcceptedStrings;
+import org.apache.lucene.util.automaton.AutomatonTestUtil;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -280,17 +286,28 @@ public abstract class BasePostingsFormat
}
private static class FieldAndTerm {
- String field;
- BytesRef term;
+ final String field;
+ final BytesRef term;
+ final long ord;
- public FieldAndTerm(String field, BytesRef term) {
+ public FieldAndTerm(String field, BytesRef term, long ord) {
this.field = field;
this.term = BytesRef.deepCopyOf(term);
+ this.ord = ord;
+ }
+ }
+
+ private static class SeedAndOrd {
+ final long seed;
+ long ord;
+
+ public SeedAndOrd(long seed) {
+ this.seed = seed;
}
}
// Holds all postings:
- private static Map<String,Map<BytesRef,Long>> fields;
+ private static Map<String,SortedMap<BytesRef,SeedAndOrd>> fields;
private static FieldInfos fieldInfos;
@@ -346,7 +363,7 @@ public abstract class BasePostingsFormat
null, DocValuesType.NUMERIC, -1, null);
fieldUpto++;
- Map<BytesRef,Long> postings = new TreeMap<>();
+ SortedMap<BytesRef,SeedAndOrd> postings = new TreeMap<>();
fields.put(field, postings);
Set<String> seenTerms = new HashSet<>();
@@ -357,7 +374,9 @@ public abstract class BasePostingsFormat
numTerms = TestUtil.nextInt(random(), 2, 20);
}
- for(int termUpto=0;termUpto<numTerms;termUpto++) {
+ while (postings.size() < numTerms) {
+ int termUpto = postings.size();
+ // Cannot contain surrogates else default Java string sort order (by UTF16 code unit) is different from Lucene:
String term = TestUtil.randomSimpleString(random());
if (seenTerms.contains(term)) {
continue;
@@ -379,7 +398,7 @@ public abstract class BasePostingsFormat
}
long termSeed = random().nextLong();
- postings.put(new BytesRef(term), termSeed);
+ postings.put(new BytesRef(term), new SeedAndOrd(termSeed));
// NOTE: sort of silly: we enum all the docs just to
// get the maxDoc
@@ -391,6 +410,12 @@ public abstract class BasePostingsFormat
}
maxDoc = Math.max(lastDoc, maxDoc);
}
+
+ // assign ords
+ long ord = 0;
+ for(SeedAndOrd ent : postings.values()) {
+ ent.ord = ord++;
+ }
}
fieldInfos = new FieldInfos(fieldInfoArray);
@@ -407,10 +432,11 @@ public abstract class BasePostingsFormat
}
allTerms = new ArrayList<>();
- for(Map.Entry<String,Map<BytesRef,Long>> fieldEnt : fields.entrySet()) {
+ for(Map.Entry<String,SortedMap<BytesRef,SeedAndOrd>> fieldEnt : fields.entrySet()) {
String field = fieldEnt.getKey();
- for(Map.Entry<BytesRef,Long> termEnt : fieldEnt.getValue().entrySet()) {
- allTerms.add(new FieldAndTerm(field, termEnt.getKey()));
+ long ord = 0;
+ for(Map.Entry<BytesRef,SeedAndOrd> termEnt : fieldEnt.getValue().entrySet()) {
+ allTerms.add(new FieldAndTerm(field, termEnt.getKey(), ord++));
}
}
@@ -487,9 +513,9 @@ public abstract class BasePostingsFormat
32, null, new IOContext(new FlushInfo(maxDoc, bytes)));
FieldsConsumer fieldsConsumer = codec.postingsFormat().fieldsConsumer(writeState);
- for(Map.Entry<String,Map<BytesRef,Long>> fieldEnt : fields.entrySet()) {
+ for(Map.Entry<String,SortedMap<BytesRef,SeedAndOrd>> fieldEnt : fields.entrySet()) {
String field = fieldEnt.getKey();
- Map<BytesRef,Long> terms = fieldEnt.getValue();
+ Map<BytesRef,SeedAndOrd> terms = fieldEnt.getValue();
FieldInfo fieldInfo = newFieldInfos.fieldInfo(field);
@@ -508,11 +534,11 @@ public abstract class BasePostingsFormat
long sumTotalTF = 0;
long sumDF = 0;
FixedBitSet seenDocs = new FixedBitSet(maxDoc);
- for(Map.Entry<BytesRef,Long> termEnt : terms.entrySet()) {
+ for(Map.Entry<BytesRef,SeedAndOrd> termEnt : terms.entrySet()) {
BytesRef term = termEnt.getKey();
- SeedPostings postings = getSeedPostings(term.utf8ToString(), termEnt.getValue(), false, maxAllowed);
+ SeedPostings postings = getSeedPostings(term.utf8ToString(), termEnt.getValue().seed, false, maxAllowed);
if (VERBOSE) {
- System.out.println(" term=" + field + ":" + term.utf8ToString() + " docFreq=" + postings.docFreq + " seed=" + termEnt.getValue());
+ System.out.println(" term=" + field + ":" + term.utf8ToString() + " docFreq=" + postings.docFreq + " seed=" + termEnt.getValue().seed);
}
PostingsConsumer postingsConsumer = termsConsumer.startTerm(term);
@@ -619,7 +645,7 @@ public abstract class BasePostingsFormat
// NOTE: can be empty list if we are using liveDocs:
SeedPostings expected = getSeedPostings(term.utf8ToString(),
- fields.get(field).get(term),
+ fields.get(field).get(term).seed,
useLiveDocs,
maxIndexOptions);
assertEquals(expected.docFreq, termsEnum.docFreq());
@@ -956,12 +982,16 @@ public abstract class BasePostingsFormat
// Test random terms/fields:
List<TermState> termStates = new ArrayList<>();
List<FieldAndTerm> termStateTerms = new ArrayList<>();
+
+ boolean supportsOrds = true;
Collections.shuffle(allTerms, random());
int upto = 0;
while (upto < allTerms.size()) {
boolean useTermState = termStates.size() != 0 && random().nextInt(5) == 1;
+ boolean useTermOrd = supportsOrds && useTermState == false && random().nextInt(5) == 1;
+
FieldAndTerm fieldAndTerm;
TermsEnum termsEnum;
@@ -971,7 +1001,11 @@ public abstract class BasePostingsFormat
// Seek by random field+term:
fieldAndTerm = allTerms.get(upto++);
if (VERBOSE) {
- System.out.println("\nTEST: seek to term=" + fieldAndTerm.field + ":" + fieldAndTerm.term.utf8ToString() );
+ if (useTermOrd) {
+ System.out.println("\nTEST: seek to term=" + fieldAndTerm.field + ":" + fieldAndTerm.term.utf8ToString() + " using ord=" + fieldAndTerm.ord);
+ } else {
+ System.out.println("\nTEST: seek to term=" + fieldAndTerm.field + ":" + fieldAndTerm.term.utf8ToString() );
+ }
}
} else {
// Seek by previous saved TermState
@@ -988,11 +1022,38 @@ public abstract class BasePostingsFormat
termsEnum = terms.iterator(null);
if (!useTermState) {
- assertTrue(termsEnum.seekExact(fieldAndTerm.term));
+ if (useTermOrd) {
+ // Try seek by ord sometimes:
+ try {
+ termsEnum.seekExact(fieldAndTerm.ord);
+ } catch (UnsupportedOperationException uoe) {
+ supportsOrds = false;
+ assertTrue(termsEnum.seekExact(fieldAndTerm.term));
+ }
+ } else {
+ assertTrue(termsEnum.seekExact(fieldAndTerm.term));
+ }
} else {
termsEnum.seekExact(fieldAndTerm.term, termState);
}
+ long termOrd;
+ if (supportsOrds) {
+ try {
+ termOrd = termsEnum.ord();
+ } catch (UnsupportedOperationException uoe) {
+ supportsOrds = false;
+ termOrd = -1;
+ }
+ } else {
+ termOrd = -1;
+ }
+
+ if (termOrd != -1) {
+ // PostingsFormat supports ords
+ assertEquals(fieldAndTerm.ord, termsEnum.ord());
+ }
+
boolean savedTermState = false;
if (options.contains(Option.TERM_STATE) && !useTermState && random().nextInt(5) == 1) {
@@ -1037,6 +1098,71 @@ public abstract class BasePostingsFormat
alwaysTestMax);
}
}
+
+ // Test Terms.intersect:
+ for(String field : fields.keySet()) {
+ while (true) {
+ Automaton a = AutomatonTestUtil.randomAutomaton(random());
+ CompiledAutomaton ca = new CompiledAutomaton(a);
+ if (ca.type != CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {
+ // Keep retrying until we get an A that will really "use" the PF's intersect code:
+ continue;
+ }
+ // System.out.println("A:\n" + a.toDot());
+
+ BytesRef startTerm = null;
+ if (random().nextBoolean()) {
+ RandomAcceptedStrings ras = new RandomAcceptedStrings(a);
+ for (int iter=0;iter<100;iter++) {
+ int[] codePoints = ras.getRandomAcceptedString(random());
+ if (codePoints.length == 0) {
+ continue;
+ }
+ startTerm = new BytesRef(UnicodeUtil.newString(codePoints, 0, codePoints.length));
+ break;
+ }
+ // Don't allow empty string startTerm:
+ if (startTerm == null) {
+ continue;
+ }
+ }
+ TermsEnum intersected = fieldsSource.terms(field).intersect(ca, startTerm);
+
+ Set<BytesRef> intersectedTerms = new HashSet<BytesRef>();
+ BytesRef term;
+ while ((term = intersected.next()) != null) {
+ if (startTerm != null) {
+ // NOTE: not <=
+ assertTrue(startTerm.compareTo(term) < 0);
+ }
+ intersectedTerms.add(BytesRef.deepCopyOf(term));
+ verifyEnum(threadState,
+ field,
+ term,
+ intersected,
+ maxTestOptions,
+ maxIndexOptions,
+ options,
+ alwaysTestMax);
+ }
+
+ if (ca.runAutomaton == null) {
+ assertTrue(intersectedTerms.isEmpty());
+ } else {
+ for(BytesRef term2 : fields.get(field).keySet()) {
+ boolean expected;
+ if (startTerm != null && startTerm.compareTo(term2) >= 0) {
+ expected = false;
+ } else {
+ expected = ca.runAutomaton.run(term2.bytes, term2.offset, term2.length);
+ }
+ assertEquals("term=" + term2, expected, intersectedTerms.contains(term2));
+ }
+ }
+
+ break;
+ }
+ }
}
private void testFields(Fields fields) throws Exception {
@@ -1155,4 +1281,89 @@ public abstract class BasePostingsFormat
}
}
}
+
+ public void testJustEmptyField() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ iwc.setCodec(getCodec());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ doc.add(newStringField("", "something", Field.Store.NO));
+ iw.addDocument(doc);
+ DirectoryReader ir = iw.getReader();
+ AtomicReader ar = getOnlySegmentReader(ir);
+ Fields fields = ar.fields();
+ int fieldCount = fields.size();
+ // -1 is allowed, if the codec doesn't implement fields.size():
+ assertTrue(fieldCount == 1 || fieldCount == -1);
+ Terms terms = ar.terms("");
+ assertNotNull(terms);
+ TermsEnum termsEnum = terms.iterator(null);
+ assertNotNull(termsEnum.next());
+ assertEquals(termsEnum.term(), new BytesRef("something"));
+ assertNull(termsEnum.next());
+ ir.close();
+ iw.close();
+ dir.close();
+ }
+
+ public void testEmptyFieldAndEmptyTerm() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ iwc.setCodec(getCodec());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ doc.add(newStringField("", "", Field.Store.NO));
+ iw.addDocument(doc);
+ DirectoryReader ir = iw.getReader();
+ AtomicReader ar = getOnlySegmentReader(ir);
+ Fields fields = ar.fields();
+ int fieldCount = fields.size();
+ // -1 is allowed, if the codec doesn't implement fields.size():
+ assertTrue(fieldCount == 1 || fieldCount == -1);
+ Terms terms = ar.terms("");
+ assertNotNull(terms);
+ TermsEnum termsEnum = terms.iterator(null);
+ assertNotNull(termsEnum.next());
+ assertEquals(termsEnum.term(), new BytesRef(""));
+ assertNull(termsEnum.next());
+ ir.close();
+ iw.close();
+ dir.close();
+ }
+
+ // tests that ghost fields still work
+ // TODO: can this be improved?
+ public void testGhosts() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
+ iwc.setCodec(getCodec());
+ iwc.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ Document doc = new Document();
+ iw.addDocument(doc);
+ doc.add(newStringField("ghostField", "something", Field.Store.NO));
+ iw.addDocument(doc);
+ iw.forceMerge(1);
+ iw.deleteDocuments(new Term("ghostField", "something")); // delete the only term for the field
+ iw.forceMerge(1);
+ DirectoryReader ir = iw.getReader();
+ AtomicReader ar = getOnlySegmentReader(ir);
+ Fields fields = ar.fields();
+ // Ghost busting terms dict impls will have
+ // fields.size() == 0; all others must be == 1:
+ assertTrue(fields.size() <= 1);
+ Terms terms = fields.terms("ghostField");
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator(null);
+ BytesRef term = termsEnum.next();
+ if (term != null) {
+ DocsEnum docsEnum = termsEnum.docs(null, null);
+ assertTrue(docsEnum.nextDoc() == DocsEnum.NO_MORE_DOCS);
+ }
+ }
+ ir.close();
+ iw.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Mon Jul 21 09:24:01 2014
@@ -692,7 +692,7 @@ public class MockDirectoryWrapper extend
if (LuceneTestCase.VERBOSE) {
System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex");
}
- TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose());
+ TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose(), true);
// TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
if (assertNoUnreferencedFilesOnClose) {
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java?rev=1612213&r1=1612212&r2=1612213&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java Mon Jul 21 09:24:01 2014
@@ -28,7 +28,6 @@ import java.io.PrintStream;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.CharBuffer;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
@@ -194,9 +193,16 @@ public final class TestUtil {
}
public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors) throws IOException {
+ return checkIndex(dir, crossCheckTermVectors, false);
+ }
+
+ /** If failFast is true, then throw the first exception when index corruption is hit, instead of moving on to other fields/segments to
+ * look for any other corruption. */
+ public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors, boolean failFast) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.setCrossCheckTermVectors(crossCheckTermVectors);
+ checker.setFailFast(failFast);
checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
CheckIndex.Status indexStatus = checker.checkIndex(null);
if (indexStatus == null || indexStatus.clean == false) {
@@ -224,24 +230,14 @@ public final class TestUtil {
PrintStream infoStream = new PrintStream(bos, false, IOUtils.UTF_8);
reader.checkIntegrity();
- FieldNormStatus fieldNormStatus = CheckIndex.testFieldNorms(reader, infoStream);
- TermIndexStatus termIndexStatus = CheckIndex.testPostings(reader, infoStream);
- StoredFieldStatus storedFieldStatus = CheckIndex.testStoredFields(reader, infoStream);
- TermVectorStatus termVectorStatus = CheckIndex.testTermVectors(reader, infoStream, false, crossCheckTermVectors);
- DocValuesStatus docValuesStatus = CheckIndex.testDocValues(reader, infoStream);
+ FieldNormStatus fieldNormStatus = CheckIndex.testFieldNorms(reader, infoStream, true);
+ TermIndexStatus termIndexStatus = CheckIndex.testPostings(reader, infoStream, false, true);
+ StoredFieldStatus storedFieldStatus = CheckIndex.testStoredFields(reader, infoStream, true);
+ TermVectorStatus termVectorStatus = CheckIndex.testTermVectors(reader, infoStream, false, crossCheckTermVectors, true);
+ DocValuesStatus docValuesStatus = CheckIndex.testDocValues(reader, infoStream, true);
- if (fieldNormStatus.error != null ||
- termIndexStatus.error != null ||
- storedFieldStatus.error != null ||
- termVectorStatus.error != null ||
- docValuesStatus.error != null) {
- System.out.println("CheckReader failed");
+ if (LuceneTestCase.INFOSTREAM) {
System.out.println(bos.toString(IOUtils.UTF_8));
- throw new RuntimeException("CheckReader failed");
- } else {
- if (LuceneTestCase.INFOSTREAM) {
- System.out.println(bos.toString(IOUtils.UTF_8));
- }
}
}