You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/05/25 10:30:56 UTC
svn commit: r1342526 - in /lucene/dev/branches/LUCENE-2878/lucene:
core/src/java/org/apache/lucene/search/
core/src/java/org/apache/lucene/search/positions/
core/src/java/org/apache/lucene/search/spans/
core/src/test/org/apache/lucene/search/positions/...
Author: simonw
Date: Fri May 25 08:30:55 2012
New Revision: 1342526
URL: http://svn.apache.org/viewvc?rev=1342526&view=rev
Log:
LUCENE-2878: Expose Interval Offsets via PositionInterval
Added:
lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java
Modified:
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java
lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java
lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java
lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/TermScorer.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/TermScorer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/TermScorer.java Fri May 25 08:30:55 2012
@@ -118,6 +118,8 @@ final class TermScorer extends Scorer {
public PositionInterval next() throws IOException {
if (--positionsPending >= 0) {
interval.begin = interval.end = docsAndPos.nextPosition();
+ interval.offsetBegin = docsAndPos.startOffset();
+ interval.offsetEnd = docsAndPos.endOffset();
return interval;
}
positionsPending = 0;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java Fri May 25 08:30:55 2012
@@ -30,10 +30,10 @@ public final class BlockPositionIterator
private final PositionIntervalIterator[] iterators;
private static final PositionInterval INFINITE_INTERVAL = new PositionInterval(
- Integer.MIN_VALUE, Integer.MIN_VALUE);
+ Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1);
private final PositionInterval[] intervals;
private final PositionInterval interval = new PositionInterval(
- Integer.MIN_VALUE, Integer.MIN_VALUE);
+ Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1);
private final int[] gaps;
private final int lastIter;
@@ -111,6 +111,8 @@ public final class BlockPositionIterator
}
interval.begin = intervals[0].begin;
interval.end = intervals[lastIter].end;
+ interval.offsetBegin = intervals[0].offsetBegin;
+ interval.offsetEnd = intervals[lastIter].offsetEnd;
return interval;
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java Fri May 25 08:30:55 2012
@@ -27,7 +27,7 @@ import org.apache.lucene.util.PriorityQu
// nocommit - javadoc
abstract class IntervalQueue extends PriorityQueue<IntervalRef> {
final PositionInterval queueInterval = new PositionInterval(
- Integer.MIN_VALUE, Integer.MIN_VALUE);
+ Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1);
public void reset() {
clear();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java Fri May 25 08:30:55 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.search.position
final class IntervalQueueAnd extends IntervalQueue {
int rightExtreme = Integer.MIN_VALUE;
+ int rightExtremeOffset = Integer.MIN_VALUE;
public IntervalQueueAnd(int size) {
super(size);
@@ -34,10 +35,12 @@ final class IntervalQueueAnd extends Int
queueInterval.begin = Integer.MIN_VALUE;
queueInterval.end = Integer.MIN_VALUE;
rightExtreme = Integer.MIN_VALUE;
+ rightExtremeOffset = Integer.MIN_VALUE;
}
public void updateRightExtreme(PositionInterval interval) {
- rightExtreme = Math.max(rightExtreme, Math.max(interval.end, interval.end));
+ rightExtreme = Math.max(rightExtreme, interval.end);
+ rightExtremeOffset = Math.max(rightExtremeOffset, interval.offsetEnd);
}
public boolean topContainsQueueInterval() {
@@ -49,7 +52,9 @@ final class IntervalQueueAnd extends Int
public void updateQueueInterval() {
PositionInterval interval = top().interval;
queueInterval.begin = interval.begin;
+ queueInterval.offsetBegin = interval.offsetBegin;
queueInterval.end = rightExtreme;
+ queueInterval.offsetEnd = rightExtremeOffset;
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java Fri May 25 08:30:55 2012
@@ -39,7 +39,9 @@ final class IntervalQueueOr extends Inte
public void updateQueueInterval() {
PositionInterval interval = top().interval;
queueInterval.begin = interval.begin;
+ queueInterval.offsetBegin = interval.offsetBegin;
queueInterval.end = interval.end;
+ queueInterval.offsetEnd = interval.offsetEnd;
}
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java Fri May 25 08:30:55 2012
@@ -26,11 +26,11 @@ public final class OrderedConjunctionPos
private final PositionIntervalIterator[] iterators;
private static final PositionInterval INFINITE_INTERVAL = new PositionInterval(
- Integer.MIN_VALUE, Integer.MIN_VALUE);
+ Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1);
private final PositionInterval[] intervals;
private final int lastIter;
private final PositionInterval interval = new PositionInterval(
- Integer.MAX_VALUE, Integer.MAX_VALUE);
+ Integer.MAX_VALUE, Integer.MAX_VALUE, -1, -1);
private int index = 1;
public OrderedConjunctionPositionIterator(PositionIntervalIterator other) {
@@ -52,6 +52,8 @@ public final class OrderedConjunctionPos
interval.begin = Integer.MAX_VALUE;
interval.end = Integer.MAX_VALUE;
+ interval.offsetBegin = -1;
+ interval.offsetEnd = -1;
int b = Integer.MAX_VALUE;
while (true) {
while (true) {
@@ -74,6 +76,8 @@ public final class OrderedConjunctionPos
}
interval.begin = intervals[0].begin;
interval.end = intervals[lastIter].end;
+ interval.offsetBegin = intervals[0].offsetBegin;
+ interval.offsetEnd = intervals[lastIter].offsetEnd;
b = intervals[lastIter].begin;
index = 1;
intervals[0] = iterators[0].next();
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java Fri May 25 08:30:55 2012
@@ -50,7 +50,7 @@ public class PositionFilterQuery extends
public Query rewrite(IndexReader reader) throws IOException {
PositionFilterQuery clone = null;
- Query rewritten = (Query) inner.rewrite(reader);
+ Query rewritten = inner.rewrite(reader);
if (rewritten != inner) {
clone = (PositionFilterQuery) this.clone();
clone.inner = rewritten;
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java Fri May 25 08:30:55 2012
@@ -16,11 +16,11 @@ package org.apache.lucene.search.positio
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import java.io.IOException;
-
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
+import java.io.IOException;
+
/**
*
* @lucene.experimental
@@ -89,14 +89,18 @@ public abstract class PositionIntervalIt
public int begin;
public int end;
+ public int offsetBegin;
+ public int offsetEnd;
- public PositionInterval(int begin, int end) {
+ public PositionInterval(int begin, int end, int offsetBegin, int offsetEnd) {
this.begin = begin;
this.end = end;
+ this.offsetBegin = offsetBegin;
+ this.offsetEnd = offsetEnd;
}
public PositionInterval() {
- this(0, 0);
+ this(0, 0, -1, -1);
}
public boolean nextPayload(BytesRef ref) throws IOException {
@@ -122,7 +126,7 @@ public abstract class PositionIntervalIt
@Override
public String toString() {
- return "PositionInterval [begin=" + begin + ", end=" + end + "]";
+ return "PositionInterval [begin=" + begin + "(" + offsetBegin + "), end=" + end + "(" + offsetEnd + ")]";
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java Fri May 25 08:30:55 2012
@@ -58,7 +58,7 @@ public class MockSpanQuery extends SpanQ
AtomicReaderContext topReaderContext = context.reader().getTopReaderContext();
Weight weight = other.createWeight(new IndexSearcher(topReaderContext));
- Scorer scorer = weight.scorer((AtomicReaderContext) topReaderContext, true, false, acceptDocs);
+ Scorer scorer = weight.scorer(topReaderContext, true, false, acceptDocs);
if (scorer == null) {
return EMPTY_SPANS;
}
Added: lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java?rev=1342526&view=auto
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java (added)
+++ lucene/dev/branches/LUCENE-2878/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java Fri May 25 08:30:55 2012
@@ -0,0 +1,206 @@
+package org.apache.lucene.search.positions;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
+import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
+import org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat;
+import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+public class TestPositionOffsets extends LuceneTestCase {
+
+ // What am I testing here?
+ // - can get offsets out of a basic TermQuery, and a more complex BooleanQuery
+ // - if offsets are not stored, then we get -1 returned
+
+ IndexWriterConfig iwc;
+
+ public void setUp() throws Exception {
+ super.setUp();
+
+ // Currently only SimpleText and Lucene40 can index offsets into postings:
+ String codecName = Codec.getDefault().getName();
+ assumeTrue("Codec does not support offsets: " + codecName,
+ codecName.equals("SimpleText") ||
+ codecName.equals("Lucene40"));
+
+ iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+
+ if (codecName.equals("Lucene40")) {
+ // Sep etc are not implemented
+ switch(random().nextInt(4)) {
+ case 0: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); break;
+ case 1: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); break;
+ case 2: iwc.setCodec(_TestUtil.alwaysPostingsFormat(
+ new Pulsing40PostingsFormat(_TestUtil.nextInt(random(), 1, 3)))); break;
+ case 3: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat())); break;
+ }
+ }
+ }
+
+
+ private static void addDocs(RandomIndexWriter writer, boolean withOffsets) throws IOException {
+ FieldType fieldType = TextField.TYPE_STORED;
+ if (withOffsets) {
+ fieldType = new FieldType(fieldType);
+ fieldType.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ }
+ Document doc = new Document();
+ doc.add(newField(
+ "field",
+ "Pease porridge hot! Pease porridge cold! Pease porridge in the pot nine days old! Some like it hot, some"
+ + " like it cold, Some like it in the pot nine days old! Pease porridge hot! Pease porridge cold!",
+ fieldType));
+ writer.addDocument(doc);
+ }
+
+ public void testTermQueryWithOffsets() throws IOException {
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, iwc);
+ addDocs(writer, true);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ writer.close();
+ Query query = new TermQuery(new Term("field", "porridge"));
+
+ Weight weight = query.createWeight(searcher);
+ IndexReaderContext topReaderContext = searcher.getTopReaderContext();
+ AtomicReaderContext[] leaves = topReaderContext.leaves();
+ assertEquals(1, leaves.length);
+ Scorer scorer = weight.scorer(leaves[0],
+ true, true, leaves[0].reader().getLiveDocs());
+
+ int nextDoc = scorer.nextDoc();
+ assertEquals(0, nextDoc);
+ PositionIntervalIterator positions = scorer.positions(false, true);
+ int[] startOffsets = new int[] { 6, 26, 47, 164, 184 };
+ int[] endOffsets = new int[] { 14, 34, 55, 172, 192 };
+
+ assertEquals(0, positions.advanceTo(nextDoc));
+ for (int i = 0; i < startOffsets.length; i++) {
+ PositionIntervalIterator.PositionInterval interval = positions.next();
+ assertEquals(startOffsets[i], interval.offsetBegin);
+ assertEquals(endOffsets[i], interval.offsetEnd);
+ }
+
+ assertNull(positions.next());
+
+ reader.close();
+ directory.close();
+ }
+
+ public void testTermQueryWithoutOffsets() throws IOException {
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, iwc);
+ addDocs(writer, false);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ writer.close();
+ Query query = new TermQuery(new Term("field", "porridge"));
+
+ Weight weight = query.createWeight(searcher);
+ IndexReaderContext topReaderContext = searcher.getTopReaderContext();
+ AtomicReaderContext[] leaves = topReaderContext.leaves();
+ assertEquals(1, leaves.length);
+ Scorer scorer = weight.scorer(leaves[0],
+ true, true, leaves[0].reader().getLiveDocs());
+
+ int nextDoc = scorer.nextDoc();
+ assertEquals(0, nextDoc);
+ PositionIntervalIterator positions = scorer.positions(false, false);
+ int[] startOffsets = new int[] { -1, -1, -1, -1, -1 };
+ int[] endOffsets = new int[] { -1, -1, -1, -1, -1 };
+
+ assertEquals(0, positions.advanceTo(nextDoc));
+ for (int i = 0; i < startOffsets.length; i++) {
+ PositionIntervalIterator.PositionInterval interval = positions.next();
+ assertEquals(startOffsets[i], interval.offsetBegin);
+ assertEquals(endOffsets[i], interval.offsetEnd);
+ }
+
+ assertNull(positions.next());
+
+ reader.close();
+ directory.close();
+ }
+
+ public void testBooleanQueryWithOffsets() throws IOException {
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory, iwc);
+ addDocs(writer, true);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ writer.close();
+ BooleanQuery query = new BooleanQuery();
+ query.add(new BooleanClause(new TermQuery(new Term("field", "porridge")), BooleanClause.Occur.MUST));
+ query.add(new BooleanClause(new TermQuery(new Term("field", "nine")), BooleanClause.Occur.MUST));
+
+ Weight weight = query.createWeight(searcher);
+ IndexReaderContext topReaderContext = searcher.getTopReaderContext();
+ AtomicReaderContext[] leaves = topReaderContext.leaves();
+ assertEquals(1, leaves.length);
+ Scorer scorer = weight.scorer(leaves[0],
+ true, true, leaves[0].reader().getLiveDocs());
+
+ int nextDoc = scorer.nextDoc();
+ assertEquals(0, nextDoc);
+ PositionIntervalIterator positions = scorer.positions(false, true);
+ int[] startOffsetsConj = new int[] { 6, 26, 47, 67, 143};
+ int[] endOffsetsConj = new int[] { 71, 71, 71, 172, 172};
+ assertEquals(0, positions.advanceTo(nextDoc));
+ PositionIntervalIterator.PositionInterval interval;
+ int i = 0;
+ while((interval = positions.next()) != null) {
+ assertEquals(startOffsetsConj[i], interval.offsetBegin);
+ assertEquals(endOffsetsConj[i], interval.offsetEnd);
+ i++;
+ }
+ assertEquals(i, startOffsetsConj.length);
+ assertNull(positions.next());
+
+ reader.close();
+ directory.close();
+ }
+
+}
\ No newline at end of file
Modified: lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java Fri May 25 08:30:55 2012
@@ -49,12 +49,10 @@ public class PosTokenStream extends Toke
// the index of the current position interval
private PositionInterval pos = null;
- private final PositionOffsetMapper pom;
- public PosTokenStream (String text, PositionIntervalIterator positions, PositionOffsetMapper pom) {
+ public PosTokenStream (String text, PositionIntervalIterator positions) {
this.text = text;
this.positions = positions;
- this.pom = pom;
}
@Override
@@ -64,8 +62,9 @@ public class PosTokenStream extends Toke
return false;
}
int b, e;
- b = pom.getStartOffset(pos.begin);
- e = pom.getEndOffset(pos.end);
+ b = pos.offsetBegin;
+ e = pos.offsetEnd;
+ assert b >=0;
termAtt.append(text, b, e);
offsetAtt.setOffset(b, e);
posIncrAtt.setPositionIncrement(1);
Modified: lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java?rev=1342526&r1=1342525&r2=1342526&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java Fri May 25 08:30:55 2012
@@ -6,12 +6,16 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
+import org.apache.lucene.codecs.memory.MemoryPostingsFormat;
+import org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat;
+import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -29,266 +33,314 @@ import org.apache.lucene.search.highligh
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.search.highlight.SimpleFragmenter;
import org.apache.lucene.search.highlight.TextFragment;
+import org.apache.lucene.search.positions.BlockPositionIterator;
import org.apache.lucene.search.positions.PositionFilterQuery;
-import org.apache.lucene.search.positions.TestBlockPositionsIterator.BlockPositionIteratorFilter;
+import org.apache.lucene.search.positions.PositionIntervalIterator;
+import org.apache.lucene.search.positions.PositionIntervalIterator.PositionIntervalFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.junit.Ignore;
/**
- * TODO: FIX THIS TEST
- * Phrase and Span Queries
- * positions callback API
+ * TODO: FIX THIS TEST Phrase and Span Queries positions callback API
*/
public class PosHighlighterTest extends LuceneTestCase {
- protected final static String F="f";
+ protected final static String F = "f";
protected Analyzer analyzer;
protected Directory dir;
protected IndexSearcher searcher;
+ private IndexWriterConfig iwc;
- private static final String PORRIDGE_VERSE =
- "Pease porridge hot! Pease porridge cold! Pease porridge in the pot nine days old! Some like it hot, some"
- + " like it cold, Some like it in the pot nine days old! Pease porridge hot! Pease porridge cold!";
+ private static final String PORRIDGE_VERSE = "Pease porridge hot! Pease porridge cold! Pease porridge in the pot nine days old! Some like it hot, some"
+ + " like it cold, Some like it in the pot nine days old! Pease porridge hot! Pease porridge cold!";
- @Override
public void setUp() throws Exception {
super.setUp();
- analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ // Currently only SimpleText and Lucene40 can index offsets into postings:
+ String codecName = Codec.getDefault().getName();
+ assumeTrue("Codec does not support offsets: " + codecName,
+ codecName.equals("SimpleText") || codecName.equals("Lucene40"));
+ iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)).setOpenMode(OpenMode.CREATE);
+
+ if (codecName.equals("Lucene40")) {
+ // Sep etc are not implemented
+ switch(random().nextInt(4)) {
+ case 0: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); break;
+ case 1: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); break;
+ case 2: iwc.setCodec(_TestUtil.alwaysPostingsFormat(
+ new Pulsing40PostingsFormat(_TestUtil.nextInt(random(), 1, 3)))); break;
+ case 3: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat())); break;
+ }
+ }
+ analyzer = iwc.getAnalyzer();
dir = newDirectory();
}
- @Override
- public void tearDown() throws Exception {
- if( searcher != null ){
+ public void close() throws IOException {
+ if (searcher != null) {
searcher.getIndexReader().close();
searcher = null;
}
dir.close();
- super.tearDown();
}
// make several docs
- protected void insertDocs (Analyzer analyzer, String... values) throws Exception {
- IndexWriterConfig config = new IndexWriterConfig(
- TEST_VERSION_CURRENT, analyzer).setOpenMode(OpenMode.CREATE);
-
- IndexWriter writer = new IndexWriter(dir, config);
-
- for( String value: values ) {
+ protected void insertDocs(Analyzer analyzer, String... values)
+ throws Exception {
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ FieldType type = new FieldType();
+ type.setIndexed(true);
+ type.setTokenized(true);
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ type.setStored(true);
+ for (String value : values) {
Document doc = new Document();
- Field f = new Field (F, value, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
- doc.add (f);
- writer.addDocument( doc );
+ Field f = newField(F, value, type);
+ doc.add(f);
+ writer.addDocument(doc);
}
writer.close();
- if( searcher != null ){
+ if (searcher != null) {
searcher.getIndexReader().close();
}
searcher = new IndexSearcher(DirectoryReader.open(dir));
}
- private String[] doSearch(Query q) throws IOException, InvalidTokenOffsetsException {
+ private String[] doSearch(Query q) throws IOException,
+ InvalidTokenOffsetsException {
return doSearch(q, 100);
}
- private class ConstantScorer implements org.apache.lucene.search.highlight.Scorer {
-
+ private class ConstantScorer implements
+ org.apache.lucene.search.highlight.Scorer {
+
@Override
public TokenStream init(TokenStream tokenStream) throws IOException {
return tokenStream;
}
-
+
@Override
- public void startFragment(TextFragment newFragment) {
- }
-
+ public void startFragment(TextFragment newFragment) {}
+
@Override
public float getTokenScore() {
return 1;
}
-
+
@Override
public float getFragmentScore() {
return 1;
- }
+ }
}
- private String[] doSearch(Query q, int maxFragSize) throws IOException, InvalidTokenOffsetsException {
- return doSearch (q, maxFragSize, 0);
+ private String[] doSearch(Query q, int maxFragSize) throws IOException,
+ InvalidTokenOffsetsException {
+ return doSearch(q, maxFragSize, 0);
}
- private String[] doSearch(Query q, int maxFragSize, int docIndex) throws IOException, InvalidTokenOffsetsException {
- // ConstantScorer is a fragment Scorer, not a search result (document) Scorer
- Highlighter highlighter = new Highlighter (new ConstantScorer());
+
+ private String[] doSearch(Query q, int maxFragSize, int docIndex)
+ throws IOException, InvalidTokenOffsetsException {
+ // ConstantScorer is a fragment Scorer, not a search result (document)
+ // Scorer
+ Highlighter highlighter = new Highlighter(new ConstantScorer());
highlighter.setTextFragmenter(new SimpleFragmenter(maxFragSize));
PosCollector collector = new PosCollector(10);
if (q instanceof MultiTermQuery) {
- ((MultiTermQuery)q).setRewriteMethod (MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
+ ((MultiTermQuery) q)
+ .setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
}
searcher.search(q, collector);
ScorePosDoc doc = collector.docs[docIndex];
- if (doc == null)
- return null;
+ if (doc == null) return null;
String text = searcher.getIndexReader().document(doc.doc).get(F);
- PositionOffsetMapper pom = new PositionOffsetMapper ();
- // FIXME: test error cases: for non-stored fields, and fields w/no term vectors
-// searcher.getIndexReader().getTermFreqVector(doc.doc, F, pom);
+ // FIXME: test error cases: for non-stored fields, and fields w/no term
+ // vectors
+ // searcher.getIndexReader().getTermFreqVector(doc.doc, F, pom);
- TextFragment[] fragTexts = highlighter.getBestTextFragments(new PosTokenStream
- (text, new PositionIntervalArrayIterator(doc.sortedPositions(), doc.posCount), pom),
- text, false, 10);
+ TextFragment[] fragTexts = highlighter.getBestTextFragments(
+ new PosTokenStream(text, new PositionIntervalArrayIterator(doc
+ .sortedPositions(), doc.posCount)), text, false, 10);
String[] frags = new String[fragTexts.length];
for (int i = 0; i < frags.length; i++)
frags[i] = fragTexts[i].toString();
return frags;
}
- public void testTerm () throws Exception {
+ public void testTerm() throws Exception {
insertDocs(analyzer, "This is a test test");
- String frags[] = doSearch (new TermQuery(new Term(F, "test")));
- assertEquals ("This is a <B>test</B> <B>test</B>", frags[0]);
+ String frags[] = doSearch(new TermQuery(new Term(F, "test")));
+ assertEquals("This is a <B>test</B> <B>test</B>", frags[0]);
+ close();
}
- public void testSeveralSnippets () throws Exception {
- String input = "this is some long text. It has the word long in many places. In fact, it has long on some different fragments. " +
- "Let us see what happens to long in this case.";
- String gold = "this is some <B>long</B> text. It has the word <B>long</B> in many places. In fact, it has <B>long</B> on some different fragments. " +
- "Let us see what happens to <B>long</B> in this case.";
+ public void testSeveralSnippets() throws Exception {
+ String input = "this is some long text. It has the word long in many places. In fact, it has long on some different fragments. "
+ + "Let us see what happens to long in this case.";
+ String gold = "this is some <B>long</B> text. It has the word <B>long</B> in many places. In fact, it has <B>long</B> on some different fragments. "
+ + "Let us see what happens to <B>long</B> in this case.";
insertDocs(analyzer, input);
- String frags[] = doSearch (new TermQuery(new Term(F, "long")), input.length());
- assertEquals (gold, frags[0]);
+ String frags[] = doSearch(new TermQuery(new Term(F, "long")),
+ input.length());
+ assertEquals(gold, frags[0]);
+ close();
}
- public void testBooleanAnd () throws Exception {
+ public void testBooleanAnd() throws Exception {
insertDocs(analyzer, "This is a test");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "This")), Occur.MUST));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.MUST));
- String frags[] = doSearch (bq);
- assertEquals ("<B>This</B> is a <B>test</B>", frags[0]);
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.MUST));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.MUST));
+ String frags[] = doSearch(bq);
+ assertEquals("<B>This</B> is a <B>test</B>", frags[0]);
+ close();
}
- public void testBooleanAndOtherOrder () throws Exception {
+ public void testBooleanAndOtherOrder() throws Exception {
insertDocs(analyzer, "This is a test");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.MUST));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "This")), Occur.MUST));
- String frags[] = doSearch (bq);
- assertEquals ("<B>This</B> is a <B>test</B>", frags[0]);
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.MUST));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.MUST));
+ String frags[] = doSearch(bq);
+ assertEquals("<B>This</B> is a <B>test</B>", frags[0]);
+ close();
}
-
- public void testBooleanOr () throws Exception {
+
+ public void testBooleanOr() throws Exception {
insertDocs(analyzer, "This is a test");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.SHOULD));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "This")), Occur.SHOULD));
- String frags[] = doSearch (bq);
- assertEquals ("<B>This</B> is a <B>test</B>", frags[0]);
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.SHOULD));
+ String frags[] = doSearch(bq);
+ assertEquals("<B>This</B> is a <B>test</B>", frags[0]);
+ close();
}
- public void testSingleMatchScorer () throws Exception {
+ public void testSingleMatchScorer() throws Exception {
insertDocs(analyzer, "This is a test");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.SHOULD));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "notoccurringterm")), Occur.SHOULD));
- String frags[] = doSearch (bq);
- assertEquals ("This is a <B>test</B>", frags[0]);
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "notoccurringterm")),
+ Occur.SHOULD));
+ String frags[] = doSearch(bq);
+ assertEquals("This is a <B>test</B>", frags[0]);
+ close();
}
- public void testBooleanNrShouldMatch () throws Exception {
+ public void testBooleanNrShouldMatch() throws Exception {
insertDocs(analyzer, "a b c d e f g h i");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "a")), Occur.SHOULD));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "b")), Occur.SHOULD));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "no")), Occur.SHOULD));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "a")), Occur.SHOULD));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "b")), Occur.SHOULD));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "no")), Occur.SHOULD));
// This generates a ConjunctionSumScorer
bq.setMinimumNumberShouldMatch(2);
- String frags[] = doSearch (bq);
- assertEquals ("<B>a</B> <B>b</B> c d e f g h i", frags[0]);
+ String frags[] = doSearch(bq);
+ assertEquals("<B>a</B> <B>b</B> c d e f g h i", frags[0]);
// This generates no scorer
bq.setMinimumNumberShouldMatch(3);
- frags = doSearch (bq);
- assertNull (frags);
+ frags = doSearch(bq);
+ assertNull(frags);
// This generates a DisjunctionSumScorer
bq.setMinimumNumberShouldMatch(2);
- bq.add(new BooleanClause (new TermQuery(new Term(F, "c")), Occur.SHOULD));
- frags = doSearch (bq);
- assertEquals ("<B>a</B> <B>b</B> <B>c</B> d e f g h i", frags[0]);
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "c")), Occur.SHOULD));
+ frags = doSearch(bq);
+ assertEquals("<B>a</B> <B>b</B> <B>c</B> d e f g h i", frags[0]);
+ close();
}
public void testPhrase() throws Exception {
insertDocs(analyzer, "is it that this is a test, is it");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "is")), Occur.MUST));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "a")), Occur.MUST));
- PositionFilterQuery pfq = new PositionFilterQuery(bq, new BlockPositionIteratorFilter());
- String frags[] = doSearch (pfq);
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "is")), Occur.MUST));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "a")), Occur.MUST));
+ PositionFilterQuery pfq = new PositionFilterQuery(bq,
+ new BlockPositionIteratorFilter());
+ String frags[] = doSearch(pfq);
// make sure we highlight the phrase, and not the terms outside the phrase
- assertEquals ("is it that this <B>is</B> <B>a</B> test, is it", frags[0]);
+ assertEquals("is it that this <B>is</B> <B>a</B> test, is it", frags[0]);
+ close();
}
/*
* Failing ... PhraseQuery scorer needs positions()?
*/
+ @Ignore
public void testPhraseOriginal() throws Exception {
insertDocs(analyzer, "This is a test");
PhraseQuery pq = new PhraseQuery();
pq.add(new Term(F, "a"));
pq.add(new Term(F, "test"));
- String frags[] = doSearch (pq);
- assertEquals ("This is <B>a</B> <B>test</B>", frags[0]);
+ String frags[] = doSearch(pq);
+ assertEquals("This is <B>a</B> <B>test</B>", frags[0]);
+ close();
}
- public void testNestedBoolean () throws Exception {
+ public void testNestedBoolean() throws Exception {
insertDocs(analyzer, "This is a test");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.SHOULD));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD));
BooleanQuery bq2 = new BooleanQuery();
- bq2.add(new BooleanClause (new TermQuery(new Term(F, "This")), Occur.SHOULD));
- bq2.add(new BooleanClause (new TermQuery(new Term(F, "is")), Occur.SHOULD));
+ bq2.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.SHOULD));
+ bq2.add(new BooleanClause(new TermQuery(new Term(F, "is")), Occur.SHOULD));
bq.add(new BooleanClause(bq2, Occur.SHOULD));
- String frags[] = doSearch (bq);
- assertEquals ("<B>This</B> <B>is</B> a <B>test</B>", frags[0]);
+ String frags[] = doSearch(bq);
+ assertEquals("<B>This</B> <B>is</B> a <B>test</B>", frags[0]);
+ close();
}
- public void testWildcard () throws Exception {
+ public void testWildcard() throws Exception {
insertDocs(analyzer, "This is a test");
- String frags[] = doSearch (new WildcardQuery(new Term(F, "t*t")));
- assertEquals ("This is a <B>test</B>", frags[0]);
+ String frags[] = doSearch(new WildcardQuery(new Term(F, "t*t")));
+ assertEquals("This is a <B>test</B>", frags[0]);
+ close();
}
public void testMultipleDocumentsAnd() throws Exception {
- insertDocs(analyzer,
- "This document has no matches",
- PORRIDGE_VERSE,
+ insertDocs(analyzer, "This document has no matches", PORRIDGE_VERSE,
"This document has some Pease porridge in it");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "Pease")), Occur.MUST));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "porridge")), Occur.MUST));
- String frags[] = doSearch (bq, 50, 0);
- assertEquals ("<B>Pease</B> <B>porridge</B> hot! <B>Pease</B> <B>porridge</B> cold! <B>Pease</B>", frags[0]);
- frags = doSearch (bq, 50, 1);
- assertEquals ("This document has some <B>Pease</B> <B>porridge</B> in it", frags[0]);
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "Pease")), Occur.MUST));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "porridge")), Occur.MUST));
+ String frags[] = doSearch(bq, 50, 0);
+ assertEquals(
+ "<B>Pease</B> <B>porridge</B> hot! <B>Pease</B> <B>porridge</B> cold! <B>Pease</B>",
+ frags[0]);
+ frags = doSearch(bq, 50, 1);
+ assertEquals("This document has some <B>Pease</B> <B>porridge</B> in it",
+ frags[0]);
+ close();
}
- /*
- * Failing: need positions callback API since DisjunctionSumScorer consumes all of a doc's
- * positions before passing the doc to the collector.
- */
+
public void testMultipleDocumentsOr() throws Exception {
- insertDocs(analyzer,
- "This document has no matches",
- PORRIDGE_VERSE,
+ insertDocs(analyzer, "This document has no matches", PORRIDGE_VERSE,
"This document has some Pease porridge in it");
BooleanQuery bq = new BooleanQuery();
- bq.add(new BooleanClause (new TermQuery(new Term(F, "Pease")), Occur.SHOULD));
- bq.add(new BooleanClause (new TermQuery(new Term(F, "porridge")), Occur.SHOULD));
- String frags[] = doSearch (bq, 50, 0);
- assertEquals ("<B>Pease</B> <B>porridge</B> hot! <B>Pease</B> <B>porridge</B> cold! <B>Pease</B>", frags[0]);
- frags = doSearch (bq, 50, 1);
- assertEquals ("This document has some <B>Pease</B> <B>porridge</B> in it", frags[0]);
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "Pease")), Occur.SHOULD));
+ bq.add(new BooleanClause(new TermQuery(new Term(F, "porridge")),
+ Occur.SHOULD));
+ String frags[] = doSearch(bq, 50, 0);
+ assertEquals(
+ "<B>Pease</B> <B>porridge</B> hot! <B>Pease</B> <B>porridge</B> cold! <B>Pease</B>",
+ frags[0]);
+ frags = doSearch(bq, 50, 1);
+ assertEquals("This document has some <B>Pease</B> <B>porridge</B> in it",
+ frags[0]);
+ close();
}
+
+ public static class BlockPositionIteratorFilter implements PositionIntervalFilter {
+ @Override
+ public PositionIntervalIterator filter(PositionIntervalIterator iter) {
+ return new BlockPositionIterator(iter);
+ }
+
+ }
+
}