You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/03/31 10:27:46 UTC
svn commit: r1670272 [3/3] - in /lucene/dev/trunk/lucene: ./
core/src/java/org/apache/lucene/search/
core/src/java/org/apache/lucene/search/payloads/
core/src/java/org/apache/lucene/search/spans/
core/src/test/org/apache/lucene/search/ core/src/test/or...
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1670272&r1=1670271&r2=1670272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java Tue Mar 31 08:27:45 2015
@@ -22,7 +22,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.IndexWriter;
@@ -201,117 +200,55 @@ public class TestSpans extends LuceneTes
makeSpanTermQuery("t3") },
slop,
ordered);
- Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq);
+ Spans spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), snq);
- assertTrue("first range", spans.next());
- assertEquals("first doc", 11, spans.doc());
- assertEquals("first start", 0, spans.start());
- assertEquals("first end", 4, spans.end());
-
- assertTrue("second range", spans.next());
- assertEquals("second doc", 11, spans.doc());
- assertEquals("second start", 2, spans.start());
- assertEquals("second end", 6, spans.end());
+ assertEquals("first doc", 11, spans.nextDoc());
+ assertEquals("first start", 0, spans.nextStartPosition());
+ assertEquals("first end", 4, spans.endPosition());
- assertFalse("third range", spans.next());
- }
+ assertEquals("second start", 2, spans.nextStartPosition());
+ assertEquals("second end", 6, spans.endPosition());
+ tstEndSpans(spans);
+ }
public void testSpanNearUnOrdered() throws Exception {
-
//See http://www.gossamer-threads.com/lists/lucene/java-dev/52270 for discussion about this test
- SpanNearQuery snq;
- snq = new SpanNearQuery(
+ SpanNearQuery senq;
+ senq = new SpanNearQuery(
new SpanQuery[] {
makeSpanTermQuery("u1"),
makeSpanTermQuery("u2") },
0,
false);
- Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq);
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 4, spans.doc());
- assertEquals("start", 1, spans.start());
- assertEquals("end", 3, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 5, spans.doc());
- assertEquals("start", 2, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 8, spans.doc());
- assertEquals("start", 2, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 9, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 2, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 10, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 2, spans.end());
- assertTrue("Has next and it shouldn't: " + spans.doc(), spans.next() == false);
+ Spans spans = MultiSpansWrapper.wrap(reader, senq);
+ tstNextSpans(spans, 4, 1, 3);
+ tstNextSpans(spans, 5, 2, 4);
+ tstNextSpans(spans, 8, 2, 4);
+ tstNextSpans(spans, 9, 0, 2);
+ tstNextSpans(spans, 10, 0, 2);
+ tstEndSpans(spans);
SpanNearQuery u1u2 = new SpanNearQuery(new SpanQuery[]{makeSpanTermQuery("u1"),
makeSpanTermQuery("u2")}, 0, false);
- snq = new SpanNearQuery(
+ senq = new SpanNearQuery(
new SpanQuery[] {
u1u2,
makeSpanTermQuery("u2")
},
1,
false);
- spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq);
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 4, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 3, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- //unordered spans can be subsets
- assertEquals("doc", 4, spans.doc());
- assertEquals("start", 1, spans.start());
- assertEquals("end", 3, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 5, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 5, spans.doc());
- assertEquals("start", 2, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 8, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 4, spans.end());
-
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 8, spans.doc());
- assertEquals("start", 2, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 9, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 2, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 9, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 10, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 2, spans.end());
-
- assertTrue("Has next and it shouldn't", spans.next() == false);
+ spans = MultiSpansWrapper.wrap(reader, senq);
+ tstNextSpans(spans, 4, 0, 3);
+ tstNextSpans(spans, 4, 1, 3); // unordered spans can be subsets
+ tstNextSpans(spans, 5, 0, 4);
+ tstNextSpans(spans, 5, 2, 4);
+ tstNextSpans(spans, 8, 0, 4);
+ tstNextSpans(spans, 8, 2, 4);
+ tstNextSpans(spans, 9, 0, 2);
+ tstNextSpans(spans, 9, 0, 4);
+ tstNextSpans(spans, 10, 0, 2);
+ tstEndSpans(spans);
}
@@ -321,21 +258,40 @@ public class TestSpans extends LuceneTes
for (int i = 0; i < terms.length; i++) {
sqa[i] = makeSpanTermQuery(terms[i]);
}
- return MultiSpansWrapper.wrap(searcher.getTopReaderContext(), new SpanOrQuery(sqa));
+ return MultiSpansWrapper.wrap(searcher.getIndexReader(), new SpanOrQuery(sqa));
}
- private void tstNextSpans(Spans spans, int doc, int start, int end)
- throws Exception {
- assertTrue("next", spans.next());
- assertEquals("doc", doc, spans.doc());
- assertEquals("start", start, spans.start());
- assertEquals("end", end, spans.end());
+ public static void tstNextSpans(Spans spans, int doc, int start, int end) throws IOException {
+ if (spans.docID() >= doc) {
+ assertEquals("docId", doc, spans.docID());
+ } else { // nextDoc needed before testing start/end
+ if (spans.docID() >= 0) {
+ assertEquals("nextStartPosition of previous doc", Spans.NO_MORE_POSITIONS, spans.nextStartPosition());
+ assertEquals("endPosition of previous doc", Spans.NO_MORE_POSITIONS, spans.endPosition());
+ }
+ assertEquals("nextDoc", doc, spans.nextDoc());
+ if (doc != Spans.NO_MORE_DOCS) {
+ assertEquals("first startPosition", -1, spans.startPosition());
+ assertEquals("first endPosition", -1, spans.endPosition());
+ }
+ }
+ if (doc != Spans.NO_MORE_DOCS) {
+ assertEquals("nextStartPosition", start, spans.nextStartPosition());
+ assertEquals("startPosition", start, spans.startPosition());
+ assertEquals("endPosition", end, spans.endPosition());
+ }
+ }
+
+ public static void tstEndSpans(Spans spans) throws Exception {
+ if (spans != null) { // null Spans is empty
+ tstNextSpans(spans, Spans.NO_MORE_DOCS, -2, -2); // start and end positions will be ignored
+ }
}
public void testSpanOrEmpty() throws Exception {
Spans spans = orSpans(new String[0]);
- assertFalse("empty next", spans.next());
-
+ tstEndSpans(spans);
+
SpanOrQuery a = new SpanOrQuery();
SpanOrQuery b = new SpanOrQuery();
assertTrue("empty should equal", a.equals(b));
@@ -344,24 +300,7 @@ public class TestSpans extends LuceneTes
public void testSpanOrSingle() throws Exception {
Spans spans = orSpans(new String[] {"w5"});
tstNextSpans(spans, 0, 4, 5);
- assertFalse("final next", spans.next());
- }
-
- public void testSpanOrMovesForward() throws Exception {
- Spans spans = orSpans(new String[] {"w1", "xx"});
-
- spans.next();
- int doc = spans.doc();
- assertEquals(0, doc);
-
- spans.skipTo(0);
- doc = spans.doc();
-
- // LUCENE-1583:
- // according to Spans, a skipTo to the same doc or less
- // should still call next() on the underlying Spans
- assertEquals(1, doc);
-
+ tstEndSpans(spans);
}
public void testSpanOrDouble() throws Exception {
@@ -370,17 +309,15 @@ public class TestSpans extends LuceneTes
tstNextSpans(spans, 2, 3, 4);
tstNextSpans(spans, 3, 4, 5);
tstNextSpans(spans, 7, 3, 4);
- assertFalse("final next", spans.next());
+ tstEndSpans(spans);
}
- public void testSpanOrDoubleSkip() throws Exception {
+ public void testSpanOrDoubleAdvance() throws Exception {
Spans spans = orSpans(new String[] {"w5", "yy"});
- assertTrue("initial skipTo", spans.skipTo(3));
- assertEquals("doc", 3, spans.doc());
- assertEquals("start", 4, spans.start());
- assertEquals("end", 5, spans.end());
+ assertEquals("initial advance", 3, spans.advance(3));
+ tstNextSpans(spans, 3, 4, 5);
tstNextSpans(spans, 7, 3, 4);
- assertFalse("final next", spans.next());
+ tstEndSpans(spans);
}
public void testSpanOrUnused() throws Exception {
@@ -389,7 +326,7 @@ public class TestSpans extends LuceneTes
tstNextSpans(spans, 2, 3, 4);
tstNextSpans(spans, 3, 4, 5);
tstNextSpans(spans, 7, 3, 4);
- assertFalse("final next", spans.next());
+ tstEndSpans(spans);
}
public void testSpanOrTripleSameDoc() throws Exception {
@@ -400,7 +337,7 @@ public class TestSpans extends LuceneTes
tstNextSpans(spans, 11, 3, 4);
tstNextSpans(spans, 11, 4, 5);
tstNextSpans(spans, 11, 5, 6);
- assertFalse("final next", spans.next());
+ tstEndSpans(spans);
}
public void testSpanScorerZeroSloppyFreq() throws Exception {
@@ -542,11 +479,15 @@ public class TestSpans extends LuceneTes
SpanTermQuery iq = new SpanTermQuery(new Term(field, include));
SpanTermQuery eq = new SpanTermQuery(new Term(field, exclude));
SpanNotQuery snq = new SpanNotQuery(iq, eq, pre, post);
- Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq);
+ Spans spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), snq);
int i = 0;
- while (spans.next()){
- i++;
+ if (spans != null) {
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS){
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ i++;
+ }
+ }
}
return i;
}
Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansEnum.java?rev=1670272&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansEnum.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansEnum.java Tue Mar 31 08:27:45 2015
@@ -0,0 +1,187 @@
+package org.apache.lucene.search.spans;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.CheckHits;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.English;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Tests Spans (v2)
+ *
+ */
+public class TestSpansEnum extends LuceneTestCase {
+ private static IndexSearcher searcher;
+ private static IndexReader reader;
+ private static Directory directory;
+
+ static final class SimplePayloadFilter extends TokenFilter {
+ int pos;
+ final PayloadAttribute payloadAttr;
+ final CharTermAttribute termAttr;
+
+ public SimplePayloadFilter(TokenStream input) {
+ super(input);
+ pos = 0;
+ payloadAttr = input.addAttribute(PayloadAttribute.class);
+ termAttr = input.addAttribute(CharTermAttribute.class);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ payloadAttr.setPayload(new BytesRef(("pos: " + pos).getBytes(StandardCharsets.UTF_8)));
+ pos++;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ pos = 0;
+ }
+ }
+
+ static Analyzer simplePayloadAnalyzer;
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ simplePayloadAnalyzer = new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(tokenizer, new SimplePayloadFilter(tokenizer));
+ }
+ };
+
+ directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
+ newIndexWriterConfig(simplePayloadAnalyzer)
+ .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)).setMergePolicy(newLogMergePolicy()));
+ //writer.infoStream = System.out;
+ for (int i = 0; i < 10; i++) {
+ Document doc = new Document();
+ doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES));
+ writer.addDocument(doc);
+ }
+ for (int i = 100; i < 110; i++) {
+ Document doc = new Document(); // doc id 10-19 have 100-109
+ doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES));
+ writer.addDocument(doc);
+ }
+ reader = writer.getReader();
+ searcher = newSearcher(reader);
+ writer.close();
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ reader.close();
+ directory.close();
+ searcher = null;
+ reader = null;
+ directory = null;
+ simplePayloadAnalyzer = null;
+ }
+
+ private void checkHits(Query query, int[] results) throws IOException {
+ CheckHits.checkHits(random(), query, "field", searcher, results);
+ }
+
+ SpanTermQuery spanTQ(String term) {
+ return new SpanTermQuery(new Term("field", term));
+ }
+
+ @Test
+ public void testSpansEnumOr1() throws Exception {
+ SpanTermQuery t1 = spanTQ("one");
+ SpanTermQuery t2 = spanTQ("two");
+ SpanOrQuery soq = new SpanOrQuery(t1, t2);
+ checkHits(soq, new int[] {1, 2, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19});
+ }
+
+ @Test
+ public void testSpansEnumOr2() throws Exception {
+ SpanTermQuery t1 = spanTQ("one");
+ SpanTermQuery t11 = spanTQ("eleven");
+ SpanOrQuery soq = new SpanOrQuery(t1, t11);
+ checkHits(soq, new int[] {1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19});
+ }
+
+ @Test
+ public void testSpansEnumOr3() throws Exception {
+ SpanTermQuery t12 = spanTQ("twelve");
+ SpanTermQuery t11 = spanTQ("eleven");
+ SpanOrQuery soq = new SpanOrQuery(t12, t11);
+ checkHits(soq, new int[] {});
+ }
+
+ @Test
+ public void testSpansEnumOrNot1() throws Exception {
+ SpanTermQuery t1 = spanTQ("one");
+ SpanTermQuery t2 = spanTQ("two");
+ SpanOrQuery soq = new SpanOrQuery(t1, t2);
+ SpanNotQuery snq = new SpanNotQuery(soq, t1);
+ checkHits(snq, new int[] {2,12});
+ }
+
+ @Test
+ public void testSpansEnumNotBeforeAfter1() throws Exception {
+ SpanTermQuery t1 = spanTQ("one");
+ SpanTermQuery t100 = spanTQ("hundred");
+ SpanNotQuery snq = new SpanNotQuery(t100, t1, 0, 0);
+ checkHits(snq, new int[] {10, 11, 12, 13, 14, 15, 16, 17, 18, 19}); // include all "one hundred ..."
+ }
+
+ @Test
+ public void testSpansEnumNotBeforeAfter2() throws Exception {
+ SpanTermQuery t1 = spanTQ("one");
+ SpanTermQuery t100 = spanTQ("hundred");
+ SpanNotQuery snq = new SpanNotQuery(t100, t1, 1, 0);
+ checkHits(snq, new int[] {}); // exclude all "one hundred ..."
+ }
+
+ @Test
+ public void testSpansEnumNotBeforeAfter3() throws Exception {
+ SpanTermQuery t1 = spanTQ("one");
+ SpanTermQuery t100 = spanTQ("hundred");
+ SpanNotQuery snq = new SpanNotQuery(t100, t1, 0, 1);
+ checkHits(snq, new int[] {10, 12, 13, 14, 15, 16, 17, 18, 19}); // exclude "one hundred one"
+ }
+}
Modified: lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1670272&r1=1670271&r2=1670272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Tue Mar 31 08:27:45 2015
@@ -308,10 +308,11 @@ public class WeightedSpanTermExtractor {
final Spans spans = q.getSpans(context, acceptDocs, termContexts);
// collect span positions
- while (spans.next()) {
- spanPositions.add(new PositionSpan(spans.start(), spans.end() - 1));
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ spanPositions.add(new PositionSpan(spans.startPosition(), spans.endPosition() - 1));
+ }
}
-
}
if (spanPositions.size() == 0) {
Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java?rev=1670272&r1=1670271&r2=1670272&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java Tue Mar 31 08:27:45 2015
@@ -681,7 +681,7 @@ public class TestMultiTermHighlighting e
}
};
SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- Query query = new SpanNearQuery(new SpanQuery[] { childQuery }, 0, true);
+ Query query = new SpanNearQuery(new SpanQuery[] { childQuery, childQuery }, 0, false);
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
String snippets[] = highlighter.highlight("body", query, searcher, topDocs);