You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2015/05/19 11:21:25 UTC
svn commit: r1680208 [1/2] - in /lucene/dev/branches/branch_5x/lucene: ./
core/ core/src/java/org/apache/lucene/search/payloads/
core/src/java/org/apache/lucene/search/spans/
core/src/test/org/apache/lucene/search/
core/src/test/org/apache/lucene/searc...
Author: romseygeek
Date: Tue May 19 09:21:24 2015
New Revision: 1680208
URL: http://svn.apache.org/r1680208
Log:
LUCENE-6371: Add collection API to Spans, remove payload methods
Added:
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanCollector.java
- copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanCollector.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/SpanNearPayloadCheckQuery.java
- copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/SpanNearPayloadCheckQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/SpanPayloadCheckQuery.java
- copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/SpanPayloadCheckQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/BufferedSpanCollector.java
- copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/BufferedSpanCollector.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanCollector.java
- copied unchanged from r1680205, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanCollector.java
Removed:
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansPayloadOrdered.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearPayloadCheckQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPayloadCheckQuery.java
Modified:
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
lucene/dev/branches/branch_5x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_5x/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
lucene/dev/branches/branch_5x/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpanQuery.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/AssertingSpans.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Tue May 19 09:21:24 2015
@@ -160,6 +160,13 @@ API Changes
* LUCENE-6445: Two new methods in Highlighter's TokenSources; the existing
methods are now marked deprecated. (David Smiley)
+* LUCENE-6371: Payload collection from Spans is moved to a more generic
+ SpanCollector framework. Spans no longer implements .hasPayload() and
+ .getPayload() methods, and instead exposes a collect() method that allows
+ the collection of arbitrary postings information. SpanPayloadCheckQuery and
+ SpanPayloadNearCheckQuery have moved from the .spans package to the .payloads
+ package. (Alan Woodward, David Smiley, Paul Elschot)
+
Other
* LUCENE-6413: Test runner should report the number of suites completed/
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Tue May 19 09:21:24 2015
@@ -17,11 +17,6 @@ package org.apache.lucene.search.payload
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Objects;
-
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
@@ -29,8 +24,6 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
-import org.apache.lucene.search.spans.NearSpansOrdered;
-import org.apache.lucene.search.spans.NearSpansUnordered;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanScorer;
@@ -40,6 +33,11 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ToStringUtils;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Objects;
+
/**
* This class is very similar to
* {@link org.apache.lucene.search.spans.SpanNearQuery} except that it factors
@@ -55,8 +53,10 @@ import org.apache.lucene.util.ToStringUt
* @see org.apache.lucene.search.similarities.Similarity.SimScorer#computePayloadFactor(int, int, int, BytesRef)
*/
public class PayloadNearQuery extends SpanNearQuery {
+
protected String fieldName;
protected PayloadFunction function;
+ protected final PayloadSpanCollector payloadCollector = new PayloadSpanCollector();
public PayloadNearQuery(SpanQuery[] clauses, int slop, boolean inOrder) {
this(clauses, slop, inOrder, new AveragePayloadFunction());
@@ -129,17 +129,18 @@ public class PayloadNearQuery extends Sp
}
public class PayloadNearSpanWeight extends SpanWeight {
+
public PayloadNearSpanWeight(SpanQuery query, IndexSearcher searcher)
throws IOException {
- super(query, searcher);
+ super(query, searcher, payloadCollector);
}
@Override
public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
- Spans spans = query.getSpans(context, acceptDocs, termContexts);
+ Spans spans = query.getSpans(context, acceptDocs, termContexts, payloadCollector);
return (spans == null)
? null
- : new PayloadNearSpanScorer(spans, this, similarity, similarity.simScorer(stats, context));
+ : new PayloadNearSpanScorer(spans, this, similarity.simScorer(stats, context));
}
@Override
@@ -176,31 +177,11 @@ public class PayloadNearQuery extends Sp
protected float payloadScore;
private int payloadsSeen;
- protected PayloadNearSpanScorer(Spans spans, SpanWeight weight,
- Similarity similarity, Similarity.SimScorer docScorer) throws IOException {
+ protected PayloadNearSpanScorer(Spans spans, SpanWeight weight, Similarity.SimScorer docScorer) throws IOException {
super(spans, weight, docScorer);
this.spans = spans;
}
- // Get the payloads associated with all underlying subspans
- public void getPayloads(Spans[] subSpans) throws IOException {
- for (int i = 0; i < subSpans.length; i++) {
- if (subSpans[i] instanceof NearSpansOrdered) {
- if (((NearSpansOrdered) subSpans[i]).isPayloadAvailable()) {
- processPayloads(((NearSpansOrdered) subSpans[i]).getPayload(),
- subSpans[i].startPosition(), subSpans[i].endPosition());
- }
- getPayloads(((NearSpansOrdered) subSpans[i]).getSubSpans());
- } else if (subSpans[i] instanceof NearSpansUnordered) {
- if (((NearSpansUnordered) subSpans[i]).isPayloadAvailable()) {
- processPayloads(((NearSpansUnordered) subSpans[i]).getPayload(),
- subSpans[i].startPosition(), subSpans[i].endPosition());
- }
- getPayloads(((NearSpansUnordered) subSpans[i]).getSubSpans());
- }
- }
- }
-
// TODO change the whole spans api to use bytesRef, or nuke spans
BytesRef scratch = new BytesRef();
@@ -237,9 +218,9 @@ public class PayloadNearQuery extends Sp
do {
int matchLength = spans.endPosition() - startPos;
freq += docScorer.computeSlopFactor(matchLength);
- Spans[] spansArr = new Spans[1];
- spansArr[0] = spans;
- getPayloads(spansArr);
+ payloadCollector.reset();
+ spans.collect(payloadCollector);
+ processPayloads(payloadCollector.getPayloads(), startPos, spans.endPosition());
startPos = spans.nextStartPosition();
} while (startPos != Spans.NO_MORE_POSITIONS);
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java Tue May 19 09:21:24 2015
@@ -17,15 +17,6 @@ package org.apache.lucene.search.payload
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
-
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
@@ -46,6 +37,15 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.Spans;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeSet;
+
/**
* Experimental class to get set of payloads for most standard Lucene queries.
* Operates like Highlighter - IndexReader should only contain doc of interest,
@@ -187,17 +187,16 @@ public class PayloadSpanUtil {
for (Term term : terms) {
termContexts.put(term, TermContext.build(context, term));
}
+
+ PayloadSpanCollector collector = new PayloadSpanCollector();
for (LeafReaderContext leafReaderContext : context.leaves()) {
- final Spans spans = query.getSpans(leafReaderContext, leafReaderContext.reader().getLiveDocs(), termContexts);
+ final Spans spans = query.getSpans(leafReaderContext, leafReaderContext.reader().getLiveDocs(), termContexts, collector);
if (spans != null) {
while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
- if (spans.isPayloadAvailable()) {
- Collection<byte[]> payload = spans.getPayload();
- for (byte [] bytes : payload) {
- payloads.add(bytes);
- }
- }
+ collector.reset();
+ spans.collect(collector);
+ payloads.addAll(collector.getPayloads());
}
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Tue May 19 09:21:24 2015
@@ -17,9 +17,6 @@ package org.apache.lucene.search.payload
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Objects;
-
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
@@ -28,6 +25,8 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.apache.lucene.search.spans.BufferedSpanCollector;
+import org.apache.lucene.search.spans.SpanCollector;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanScorer;
import org.apache.lucene.search.spans.SpanTermQuery;
@@ -37,6 +36,9 @@ import org.apache.lucene.search.spans.Te
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import java.io.IOException;
+import java.util.Objects;
+
/**
* This class is very similar to
* {@link org.apache.lucene.search.spans.SpanTermQuery} except that it factors
@@ -67,19 +69,52 @@ public class PayloadTermQuery extends Sp
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
- return new PayloadTermWeight(this, searcher);
+ return new PayloadTermWeight(this, searcher, new PayloadTermCollector());
+ }
+
+ protected class PayloadTermCollector implements SpanCollector {
+
+ BytesRef payload;
+
+ @Override
+ public void reset() {
+ payload = null;
+ }
+
+ @Override
+ public int requiredPostings() {
+ return PostingsEnum.PAYLOADS;
+ }
+
+ @Override
+ public void collectLeaf(PostingsEnum postings, Term term) throws IOException {
+ payload = postings.getPayload();
+ }
+
+ @Override
+ public BufferedSpanCollector buffer() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public SpanCollector bufferedCollector() {
+ throw new UnsupportedOperationException();
+ }
}
protected class PayloadTermWeight extends SpanWeight {
- public PayloadTermWeight(PayloadTermQuery query, IndexSearcher searcher)
+ final PayloadTermCollector payloadCollector;
+
+ public PayloadTermWeight(PayloadTermQuery query, IndexSearcher searcher, PayloadTermCollector collector)
throws IOException {
- super(query, searcher);
+ super(query, searcher, collector);
+ this.payloadCollector = collector;
}
@Override
public PayloadTermSpanScorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException {
- TermSpans spans = (TermSpans) query.getSpans(context, acceptDocs, termContexts);
+ TermSpans spans = (TermSpans) query.getSpans(context, acceptDocs, termContexts, payloadCollector);
return (spans == null)
? null
: new PayloadTermSpanScorer(spans, this, similarity.simScorer(stats, context));
@@ -109,29 +144,22 @@ public class PayloadTermQuery extends Sp
freq += docScorer.computeSlopFactor(matchLength);
numMatches++;
- processPayload(similarity);
+ payloadCollector.reset();
+ spans.collect(payloadCollector);
+ processPayload();
startPos = spans.nextStartPosition();
} while (startPos != Spans.NO_MORE_POSITIONS);
}
- protected void processPayload(Similarity similarity) throws IOException {
- if (spans.isPayloadAvailable()) {
- final PostingsEnum postings = termSpans.getPostings();
- payload = postings.getPayload();
- if (payload != null) {
- payloadScore = function.currentScore(docID(), term.field(),
- spans.startPosition(), spans.endPosition(), payloadsSeen, payloadScore,
- docScorer.computePayloadFactor(docID(), spans.startPosition(), spans.endPosition(), payload));
- } else {
- payloadScore = function.currentScore(docID(), term.field(),
- spans.startPosition(), spans.endPosition(), payloadsSeen, payloadScore, 1F);
- }
- payloadsSeen++;
-
- } else {
- // zero out the payload?
- }
+ protected void processPayload() throws IOException {
+
+ float payloadFactor = payloadCollector.payload == null ? 1F :
+ docScorer.computePayloadFactor(docID(), spans.startPosition(), spans.endPosition(), payloadCollector.payload);
+ payloadScore = function.currentScore(docID(), term.field(), spans.startPosition(), spans.endPosition(),
+ payloadsSeen, payloadScore, payloadFactor);
+ payloadsSeen++;
+
}
/**
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java Tue May 19 09:21:24 2015
@@ -19,7 +19,6 @@ package org.apache.lucene.search.spans;
import java.io.IOException;
import java.util.Arrays;
-import java.util.Collection;
import java.util.Objects;
abstract class ContainSpans extends ConjunctionSpans {
@@ -49,12 +48,8 @@ abstract class ContainSpans extends Conj
}
@Override
- public boolean isPayloadAvailable() throws IOException {
- return sourceSpans.isPayloadAvailable();
+ public void collect(SpanCollector collector) throws IOException {
+ sourceSpans.collect(collector);
}
- @Override
- public Collection<byte[]> getPayload() throws IOException {
- return sourceSpans.getPayload();
- }
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Tue May 19 09:21:24 2015
@@ -17,21 +17,20 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import java.util.Objects;
-
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Weight;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
/**
* <p>Wrapper to allow {@link SpanQuery} objects participate in composite
* single-field SpanQueries by 'lying' about their search field. That is,
@@ -97,8 +96,8 @@ public class FieldMaskingSpanQuery exten
// ...this is done to be more consistent with things like SpanFirstQuery
@Override
- public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
- return maskedQuery.getSpans(context, acceptDocs, termContexts);
+ public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+ return maskedQuery.getSpans(context, acceptDocs, termContexts, collector);
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java Tue May 19 09:21:24 2015
@@ -17,12 +17,11 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import org.apache.lucene.search.TwoPhaseIterator;
+
import java.io.IOException;
-import java.util.Collection;
import java.util.Objects;
-import org.apache.lucene.search.TwoPhaseIterator;
-
/**
* A {@link Spans} implementation wrapping another spans instance,
* allowing to filter spans matches easily by implementing {@link #accept}
@@ -110,17 +109,12 @@ public abstract class FilterSpans extend
return atFirstInCurrentDoc ? -1
: (startPos != NO_MORE_POSITIONS) ? in.endPosition() : NO_MORE_POSITIONS;
}
-
- @Override
- public final Collection<byte[]> getPayload() throws IOException {
- return in.getPayload();
- }
@Override
- public final boolean isPayloadAvailable() throws IOException {
- return in.isPayloadAvailable();
+ public void collect(SpanCollector collector) throws IOException {
+ in.collect(collector);
}
-
+
@Override
public final long cost() {
return in.cost();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java Tue May 19 09:21:24 2015
@@ -19,12 +19,9 @@ package org.apache.lucene.search.spans;
import java.io.IOException;
import java.util.List;
-import java.util.Collection;
/** A Spans that is formed from the ordered subspans of a SpanNearQuery
- * where the subspans do not overlap and have a maximum slop between them,
- * and that does not need to collect payloads.
- * To also collect payloads, see {@link NearSpansPayloadOrdered}.
+ * where the subspans do not overlap and have a maximum slop between them.
* <p>
* The formed spans only contains minimum slop matches.<br>
* The matching slop is computed from the distance(s) between
@@ -41,6 +38,9 @@ import java.util.Collection;
* <pre>t1 t2 .. t3 </pre>
* <pre> t1 .. t2 t3</pre>
*
+ * Because the algorithm used to minimize the size of a match consumes
+ * child Spans eagerly, this uses a BufferedSpanCollector to collect
+ * information from subspans.
*
* Expert:
* Only public for subclassing. Most implementations should not need this class
@@ -51,9 +51,13 @@ public class NearSpansOrdered extends Ne
protected int matchStart = -1;
protected int matchEnd = -1;
- public NearSpansOrdered(SpanNearQuery query, List<Spans> subSpans) throws IOException {
+ protected final SpanCollector collector;
+ protected BufferedSpanCollector buffer;
+
+ public NearSpansOrdered(SpanNearQuery query, List<Spans> subSpans, SpanCollector collector) throws IOException {
super(query, subSpans);
this.atFirstInCurrentDoc = true; // -1 startPosition/endPosition also at doc -1
+ this.collector = collector;
}
@Override
@@ -140,10 +144,15 @@ public class NearSpansOrdered extends Ne
matchStart = lastSubSpans.startPosition();
matchEnd = lastSubSpans.endPosition();
+ buffer = collector.buffer();
+ buffer.collectCandidate(subSpans[subSpans.length - 1]);
+ buffer.accept();
+
int matchSlop = 0;
int lastStart = matchStart;
for (int i = subSpans.length - 2; i >= 0; i--) {
Spans prevSpans = subSpans[i];
+ buffer.collectCandidate(prevSpans);
int prevStart = prevSpans.startPosition();
int prevEnd = prevSpans.endPosition();
@@ -160,8 +169,11 @@ public class NearSpansOrdered extends Ne
// prevSpans still before (lastStart, lastEnd)
prevStart = ppStart;
prevEnd = ppEnd;
+ buffer.collectCandidate(prevSpans);
}
+ buffer.accept();
+
assert prevStart <= matchStart;
if (matchStart > prevEnd) { // Only non overlapping spans add to slop.
matchSlop += (matchStart - prevEnd);
@@ -190,13 +202,10 @@ public class NearSpansOrdered extends Ne
}
@Override
- public Collection<byte[]> getPayload() throws IOException {
- return null;
- }
-
- @Override
- public boolean isPayloadAvailable() {
- return false;
+ public void collect(SpanCollector collector) {
+ assert collector == this.collector
+ : "You must collect using the same SpanCollector as was passed to the NearSpans constructor";
+ buffer.replay();
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java Tue May 19 09:21:24 2015
@@ -22,10 +22,7 @@ import org.apache.lucene.util.PriorityQu
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
-import java.util.Set;
-import java.util.HashSet;
/**
* Similar to {@link NearSpansOrdered}, but for the unordered case.
@@ -118,13 +115,8 @@ public class NearSpansUnordered extends
}
@Override
- public Collection<byte[]> getPayload() throws IOException {
- return in.getPayload();
- }
-
- @Override
- public boolean isPayloadAvailable() throws IOException {
- return in.isPayloadAvailable();
+ public void collect(SpanCollector collector) throws IOException {
+ in.collect(collector);
}
@Override
@@ -249,31 +241,11 @@ public class NearSpansUnordered extends
: maxEndPositionCell.endPosition();
}
-
- /**
- * WARNING: The List is not necessarily in order of the positions.
- * @return Collection of <code>byte[]</code> payloads
- * @throws IOException if there is a low-level I/O error
- */
@Override
- public Collection<byte[]> getPayload() throws IOException {
- Set<byte[]> matchPayload = new HashSet<>();
+ public void collect(SpanCollector collector) throws IOException {
for (SpansCell cell : subSpanCells) {
- if (cell.isPayloadAvailable()) {
- matchPayload.addAll(cell.getPayload());
- }
- }
- return matchPayload;
- }
-
- @Override
- public boolean isPayloadAvailable() throws IOException {
- for (SpansCell cell : subSpanCells) {
- if (cell.isPayloadAvailable()) {
- return true;
- }
+ cell.collect(collector);
}
- return false;
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainQuery.java Tue May 19 09:21:24 2015
@@ -17,19 +17,19 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.Set;
-import java.util.Objects;
-
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Bits;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
abstract class SpanContainQuery extends SpanQuery implements Cloneable {
SpanQuery big;
SpanQuery little;
@@ -55,12 +55,12 @@ abstract class SpanContainQuery extends
little.extractTerms(terms);
}
- ArrayList<Spans> prepareConjunction(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
- Spans bigSpans = big.getSpans(context, acceptDocs, termContexts);
+ ArrayList<Spans> prepareConjunction(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+ Spans bigSpans = big.getSpans(context, acceptDocs, termContexts, collector);
if (bigSpans == null) {
return null;
}
- Spans littleSpans = little.getSpans(context, acceptDocs, termContexts);
+ Spans littleSpans = little.getSpans(context, acceptDocs, termContexts, collector);
if (littleSpans == null) {
return null;
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanContainingQuery.java Tue May 19 09:21:24 2015
@@ -17,15 +17,15 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Map;
-import java.util.ArrayList;
-
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
import org.apache.lucene.util.Bits;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+
/** Keep matches that contain another Spans. */
public class SpanContainingQuery extends SpanContainQuery {
/** Construct a SpanContainingQuery matching spans from <code>big</code>
@@ -54,8 +54,8 @@ public class SpanContainingQuery extends
* The payload is from the spans of <code>big</code>.
*/
@Override
- public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
- ArrayList<Spans> containerContained = prepareConjunction(context, acceptDocs, termContexts);
+ public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+ ArrayList<Spans> containerContained = prepareConjunction(context, acceptDocs, termContexts, collector);
if (containerContained == null) {
return null;
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java Tue May 19 09:21:24 2015
@@ -17,22 +17,22 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import java.util.Objects;
-
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
+import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TopTermsRewrite;
import org.apache.lucene.search.ScoringRewrite;
-import org.apache.lucene.search.BooleanClause.Occur; // javadocs only
+import org.apache.lucene.search.TopTermsRewrite;
import org.apache.lucene.util.Bits;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
/**
* Wraps any {@link MultiTermQuery} as a {@link SpanQuery},
* so it can be nested within other SpanQuery classes.
@@ -99,7 +99,7 @@ public class SpanMultiTermQueryWrapper<Q
}
@Override
- public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+ public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
throw new UnsupportedOperationException("Query should have been rewritten");
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java Tue May 19 09:21:24 2015
@@ -17,15 +17,8 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.Terms;
@@ -33,6 +26,13 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
/** Matches spans which are near one another. One can specify <i>slop</i>, the
* maximum number of intervening unmatched positions, as well as whether
* matches are required to be in-order.
@@ -118,11 +118,17 @@ public class SpanNearQuery extends SpanQ
}
@Override
- public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
- ArrayList<Spans> subSpans = new ArrayList<>(clauses.size());
+ public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+ Terms terms = context.reader().terms(field);
+ if (terms == null) {
+ return null; // field does not exist
+ }
+
+ ArrayList<Spans> subSpans = new ArrayList<>(clauses.size());
+ SpanCollector subSpanCollector = inOrder ? collector.bufferedCollector() : collector;
for (SpanQuery seq : clauses) {
- Spans subSpan = seq.getSpans(context, acceptDocs, termContexts);
+ Spans subSpan = seq.getSpans(context, acceptDocs, termContexts, subSpanCollector);
if (subSpan != null) {
subSpans.add(subSpan);
} else {
@@ -130,15 +136,9 @@ public class SpanNearQuery extends SpanQ
}
}
- Terms terms = context.reader().terms(field);
- if (terms == null) {
- return null; // field does not exist
- }
// all NearSpans require at least two subSpans
- return (! inOrder) ? new NearSpansUnordered(this, subSpans)
- : collectPayloads && terms.hasPayloads() ? new NearSpansPayloadOrdered(this, subSpans)
- : new NearSpansOrdered(this, subSpans);
+ return (! inOrder) ? new NearSpansUnordered(this, subSpans) : new NearSpansOrdered(this, subSpans, collector);
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java Tue May 19 09:21:24 2015
@@ -105,13 +105,13 @@ public class SpanNotQuery extends SpanQu
}
@Override
- public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
- final Spans includeSpans = include.getSpans(context, acceptDocs, termContexts);
+ public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+ final Spans includeSpans = include.getSpans(context, acceptDocs, termContexts, collector);
if (includeSpans == null) {
return null;
}
- final Spans excludeSpans = exclude.getSpans(context, acceptDocs, termContexts);
+ final Spans excludeSpans = exclude.getSpans(context, acceptDocs, termContexts, collector);
if (excludeSpans == null) {
return includeSpans;
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java Tue May 19 09:21:24 2015
@@ -17,26 +17,24 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-
-import java.util.List;
-import java.util.Collection;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.ToStringUtils;
-import org.apache.lucene.search.Query;
import org.apache.lucene.search.DisiPriorityQueue;
import org.apache.lucene.search.DisiWrapper;
-import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.DisjunctionDISIApproximation;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ToStringUtils;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/** Matches the union of its clauses.
@@ -147,13 +145,13 @@ public class SpanOrQuery extends SpanQue
@Override
- public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts)
+ public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector)
throws IOException {
final ArrayList<Spans> subSpans = new ArrayList<>(clauses.size());
for (SpanQuery sq : clauses) {
- Spans spans = sq.getSpans(context, acceptDocs, termContexts);
+ Spans spans = sq.getSpans(context, acceptDocs, termContexts, collector);
if (spans != null) {
subSpans.add(spans);
}
@@ -306,17 +304,9 @@ public class SpanOrQuery extends SpanQue
}
@Override
- public Collection<byte[]> getPayload() throws IOException {
- return topPositionSpans == null
- ? null
- : topPositionSpans.isPayloadAvailable()
- ? new ArrayList<>(topPositionSpans.getPayload())
- : null;
- }
-
- @Override
- public boolean isPayloadAvailable() throws IOException {
- return (topPositionSpans != null) && topPositionSpans.isPayloadAvailable();
+ public void collect(SpanCollector collector) throws IOException {
+ if (topPositionSpans != null)
+ topPositionSpans.collect(collector);
}
@Override
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java Tue May 19 09:21:24 2015
@@ -77,8 +77,8 @@ public abstract class SpanPositionCheckQ
protected abstract AcceptStatus acceptPosition(Spans spans) throws IOException;
@Override
- public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
- Spans matchSpans = match.getSpans(context, acceptDocs, termContexts);
+ public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+ Spans matchSpans = match.getSpans(context, acceptDocs, termContexts, collector);
return (matchSpans == null) ? null : new FilterSpans(matchSpans) {
@Override
protected AcceptStatus accept(Spans candidate) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanQuery.java Tue May 19 09:21:24 2015
@@ -17,10 +17,6 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
@@ -29,13 +25,17 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
/** Base class for span-based queries. */
public abstract class SpanQuery extends Query {
/** Expert: Returns the matches for this query in an index.
* Used internally to search for spans.
* This may return null to indicate that the SpanQuery has no results.
*/
- public abstract Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException;
+ public abstract Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException;
/**
* Extract terms from these spans.
@@ -53,7 +53,7 @@ public abstract class SpanQuery extends
@Override
public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
- return new SpanWeight(this, searcher);
+ return new SpanWeight(this, searcher, SpanCollector.NO_OP);
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Tue May 19 09:21:24 2015
@@ -17,13 +17,8 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Map;
-import java.util.Set;
-import java.util.Objects;
-
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.TermState;
@@ -32,6 +27,11 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
/** Matches spans containing a term.
* This should not be used for terms that are indexed at position Integer.MAX_VALUE.
*/
@@ -83,7 +83,7 @@ public class SpanTermQuery extends SpanQ
}
@Override
- public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) throws IOException {
+ public Spans getSpans(final LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
TermContext termContext = termContexts.get(term);
final TermState state;
if (termContext == null) {
@@ -115,7 +115,7 @@ public class SpanTermQuery extends SpanQ
final TermsEnum termsEnum = context.reader().terms(term.field()).iterator();
termsEnum.seekExact(term.bytes(), state);
- final PostingsEnum postings = termsEnum.postings(acceptDocs, null, PostingsEnum.PAYLOADS);
+ final PostingsEnum postings = termsEnum.postings(acceptDocs, null, collector.requiredPostings());
return new TermSpans(postings, term);
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java Tue May 19 09:21:24 2015
@@ -17,12 +17,6 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
@@ -37,6 +31,12 @@ import org.apache.lucene.search.similari
import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.util.Bits;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
/**
* Expert-only. Public for use by other weight implementations
*/
@@ -44,12 +44,14 @@ public class SpanWeight extends Weight {
protected final Similarity similarity;
protected final Map<Term,TermContext> termContexts;
protected final SpanQuery query;
+ protected final SpanCollector collector;
protected Similarity.SimWeight stats;
- public SpanWeight(SpanQuery query, IndexSearcher searcher) throws IOException {
+ public SpanWeight(SpanQuery query, IndexSearcher searcher, SpanCollector collector) throws IOException {
super(query);
this.similarity = searcher.getSimilarity();
this.query = query;
+ this.collector = collector;
termContexts = new HashMap<>();
TreeSet<Term> terms = new TreeSet<>();
@@ -97,7 +99,7 @@ public class SpanWeight extends Weight {
if (terms != null && terms.hasPositions() == false) {
throw new IllegalStateException("field \"" + query.getField() + "\" was indexed without position data; cannot run SpanQuery (query=" + query + ")");
}
- Spans spans = query.getSpans(context, acceptDocs, termContexts);
+ Spans spans = query.getSpans(context, acceptDocs, termContexts, collector);
return (spans == null) ? null : new SpanScorer(spans, this, similarity.simScorer(stats, context));
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanWithinQuery.java Tue May 19 09:21:24 2015
@@ -17,15 +17,15 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Map;
-import java.util.ArrayList;
-
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.TermContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
import org.apache.lucene.util.Bits;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
+
/** Keep matches that are contained within another Spans. */
public class SpanWithinQuery extends SpanContainQuery {
/** Construct a SpanWithinQuery matching spans from <code>little</code>
@@ -54,8 +54,8 @@ public class SpanWithinQuery extends Spa
* The payload is from the spans of <code>little</code>.
*/
@Override
- public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts) throws IOException {
- ArrayList<Spans> containerContained = prepareConjunction(context, acceptDocs, termContexts);
+ public Spans getSpans(final LeafReaderContext context, final Bits acceptDocs, final Map<Term,TermContext> termContexts, SpanCollector collector) throws IOException {
+ ArrayList<Spans> containerContained = prepareConjunction(context, acceptDocs, termContexts, collector);
if (containerContained == null) {
return null;
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java Tue May 19 09:21:24 2015
@@ -17,12 +17,11 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Collection;
-
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.TwoPhaseIterator;
+import java.io.IOException;
+
/** Iterates through combinations of start/end positions per-doc.
* Each start/end position represents a range of term positions within the current document.
* These are enumerated in order, by increasing document number, within that by
@@ -51,33 +50,12 @@ public abstract class Spans extends DocI
public abstract int endPosition();
/**
- * Returns the payload data for the current start/end position.
- * This is only valid after {@link #nextStartPosition()}
- * returned an available start position.
- * This method must not be called more than once after each call
- * of {@link #nextStartPosition()}. However, most payloads are loaded lazily,
- * so if the payload data for the current position is not needed,
- * this method may not be called at all for performance reasons.
- * <br>
- * Note that the return type is a collection, thus the ordering should not be relied upon.
- * <br>
- * @lucene.experimental
- *
- * @return a List of byte arrays containing the data of this payload, otherwise null if isPayloadAvailable is false
- * @throws IOException if there is a low-level I/O error
- */
- public abstract Collection<byte[]> getPayload() throws IOException;
-
- /**
- * Checks if a payload can be loaded at the current start/end position.
- * <p>
- * Payloads can only be loaded once per call to
- * {@link #nextStartPosition()}.
+ * Collect data from the current Spans
+ * @param collector a SpanCollector
*
- * @return true if there is a payload available at this start/end position
- * that can be loaded
+ * @lucene.experimental
*/
- public abstract boolean isPayloadAvailable() throws IOException;
+ public abstract void collect(SpanCollector collector) throws IOException;
/**
* Optional method: Return a {@link TwoPhaseIterator} view of this
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java Tue May 19 09:21:24 2015
@@ -16,14 +16,11 @@ package org.apache.lucene.search.spans;
*/
-import org.apache.lucene.index.Term;
import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.util.BytesRef;
import java.io.IOException;
-import java.util.Collections;
-import java.util.Collection;
import java.util.Objects;
/**
@@ -109,6 +106,7 @@ public class TermSpans extends Spans {
return postings.cost();
}
+ /*
@Override
public Collection<byte[]> getPayload() throws IOException {
final BytesRef payload = postings.getPayload();
@@ -127,6 +125,12 @@ public class TermSpans extends Spans {
public boolean isPayloadAvailable() throws IOException {
return readPayload == false && postings.getPayload() != null;
}
+ */
+
+ @Override
+ public void collect(SpanCollector collector) throws IOException {
+ collector.collectLeaf(postings, term);
+ }
@Override
public String toString() {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Tue May 19 09:21:24 2015
@@ -17,34 +17,37 @@ package org.apache.lucene.search;
* limitations under the License.
*/
-import java.io.IOException;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockPayloadAnalyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
-import org.apache.lucene.store.Directory;
+import org.apache.lucene.search.payloads.PayloadSpanCollector;
import org.apache.lucene.search.payloads.PayloadSpanUtil;
import org.apache.lucene.search.spans.MultiSpansWrapper;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.Spans;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
/**
* Term position unit test.
@@ -53,7 +56,7 @@ import org.apache.lucene.util.BytesRef;
*/
public class TestPositionIncrement extends LuceneTestCase {
- final static boolean VERBOSE = false;
+ final static boolean VERBOSE = true;
public void testSetPosition() throws Exception {
Analyzer analyzer = new Analyzer() {
@@ -238,14 +241,17 @@ public class TestPositionIncrement exten
if (VERBOSE) {
System.out.println("\ngetPayloadSpans test");
}
- Spans pspans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+ Spans pspans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, collector);
while (pspans.nextDoc() != Spans.NO_MORE_DOCS) {
while (pspans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
if (VERBOSE) {
System.out.println("doc " + pspans.docID() + ": span " + pspans.startPosition()
+ " to " + pspans.endPosition());
}
- Collection<byte[]> payloads = pspans.getPayload();
+ collector.reset();
+ pspans.collect(collector);
+ Collection<byte[]> payloads = collector.getPayloads();
sawZero |= pspans.startPosition() == 0;
for (byte[] bytes : payloads) {
count++;
@@ -256,7 +262,7 @@ public class TestPositionIncrement exten
}
}
assertTrue(sawZero);
- assertEquals(5, count);
+ assertEquals(8, count);
// System.out.println("\ngetSpans test");
Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
@@ -282,7 +288,7 @@ public class TestPositionIncrement exten
//System.out.println(s);
sawZero |= s.equals("pos: 0");
}
- assertEquals(5, count);
+ assertEquals(8, count);
assertTrue(sawZero);
writer.close();
is.getIndexReader().close();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadBasics.java Tue May 19 09:21:24 2015
@@ -17,13 +17,6 @@ package org.apache.lucene.search.payload
* limitations under the License.
*/
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.SimplePayloadFilter;
@@ -36,9 +29,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.spans.SpanNearPayloadCheckQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
-import org.apache.lucene.search.spans.SpanPayloadCheckQuery;
import org.apache.lucene.search.spans.SpanPositionRangeQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
@@ -50,6 +41,13 @@ import org.apache.lucene.util.TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
/** basic test of payload-spans */
public class TestPayloadBasics extends LuceneTestCase {
private static IndexSearcher searcher;
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadSpans.java Tue May 19 09:21:24 2015
@@ -16,32 +16,23 @@ package org.apache.lucene.search.payload
* limitations under the License.
*/
-import java.io.IOException;
-import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.payloads.PayloadHelper;
-import org.apache.lucene.search.payloads.PayloadSpanUtil;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.spans.MultiSpansWrapper;
@@ -55,6 +46,13 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
public class TestPayloadSpans extends LuceneTestCase {
private IndexSearcher searcher;
private Similarity similarity = new DefaultSimilarity();
@@ -74,14 +72,15 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq;
Spans spans;
stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "seventy"));
- spans = MultiSpansWrapper.wrap(indexReader, stq);
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+ spans = MultiSpansWrapper.wrap(indexReader, stq, collector);
assertTrue("spans is null and it shouldn't be", spans != null);
- checkSpans(spans, 100, 1, 1, 1);
+ checkSpans(spans, collector, 100, 1, 1, 1);
stq = new SpanTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "seventy"));
- spans = MultiSpansWrapper.wrap(indexReader, stq);
+ spans = MultiSpansWrapper.wrap(indexReader, stq, collector);
assertTrue("spans is null and it shouldn't be", spans != null);
- checkSpans(spans, 100, 0, 0, 0);
+ checkSpans(spans, collector, 100, 0, 0, 0);
}
public void testSpanFirst() throws IOException {
@@ -90,19 +89,20 @@ public class TestPayloadSpans extends Lu
SpanFirstQuery sfq;
match = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
sfq = new SpanFirstQuery(match, 2);
- Spans spans = MultiSpansWrapper.wrap(indexReader, sfq);
- checkSpans(spans, 109, 1, 1, 1);
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+ Spans spans = MultiSpansWrapper.wrap(indexReader, sfq, collector);
+ checkSpans(spans, collector, 109, 1, 1, 1);
//Test more complicated subclause
SpanQuery[] clauses = new SpanQuery[2];
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "hundred"));
match = new SpanNearQuery(clauses, 0, true);
sfq = new SpanFirstQuery(match, 2);
- checkSpans(MultiSpansWrapper.wrap(indexReader, sfq), 100, 2, 1, 1);
+ checkSpans(MultiSpansWrapper.wrap(indexReader, sfq, collector), collector, 100, 2, 1, 1);
match = new SpanNearQuery(clauses, 0, false);
sfq = new SpanFirstQuery(match, 2);
- checkSpans(MultiSpansWrapper.wrap(indexReader, sfq), 100, 2, 1, 1);
+ checkSpans(MultiSpansWrapper.wrap(indexReader, sfq, collector), collector, 100, 2, 1, 1);
}
@@ -124,9 +124,9 @@ public class TestPayloadSpans extends Lu
writer.addDocument(doc);
IndexReader reader = writer.getReader();
writer.close();
-
- checkSpans(MultiSpansWrapper.wrap(reader, snq), 1,new int[]{2});
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+ checkSpans(MultiSpansWrapper.wrap(reader, snq, collector), collector, 1, new int[]{2});
reader.close();
directory.close();
}
@@ -135,8 +135,10 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq;
Spans spans;
IndexSearcher searcher = getSearcher();
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+
stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "mark"));
- spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), stq);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), stq, collector);
assertNull(spans);
SpanQuery[] clauses = new SpanQuery[3];
@@ -145,9 +147,9 @@ public class TestPayloadSpans extends Lu
clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx"));
SpanNearQuery spanNearQuery = new SpanNearQuery(clauses, 12, false);
- spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery, collector);
assertTrue("spans is null and it shouldn't be", spans != null);
- checkSpans(spans, 2, new int[]{3,3});
+ checkSpans(spans, collector, 2, new int[]{3,3});
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx"));
@@ -156,10 +158,10 @@ public class TestPayloadSpans extends Lu
spanNearQuery = new SpanNearQuery(clauses, 6, true);
- spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery, collector);
assertTrue("spans is null and it shouldn't be", spans != null);
- checkSpans(spans, 1, new int[]{3});
+ checkSpans(spans, collector, 1, new int[]{3});
clauses = new SpanQuery[2];
@@ -178,10 +180,9 @@ public class TestPayloadSpans extends Lu
SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses2, 6, false);
// yy within 6 of xx within 6 of rr
-
- spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, collector);
assertTrue("spans is null and it shouldn't be", spans != null);
- checkSpans(spans, 2, new int[]{3,3});
+ checkSpans(spans, collector, 2, new int[]{3,3});
closeIndexReader.close();
directory.close();
}
@@ -208,12 +209,13 @@ public class TestPayloadSpans extends Lu
clauses3[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "np"));
clauses3[1] = snq;
-
+
+ PayloadSpanCollector collector = new PayloadSpanCollector();
SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false);
- spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, collector);
assertTrue("spans is null and it shouldn't be", spans != null);
- checkSpans(spans, 1, new int[]{3});
+ checkSpans(spans, collector, 1, new int[]{3});
closeIndexReader.close();
directory.close();
}
@@ -248,9 +250,10 @@ public class TestPayloadSpans extends Lu
SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false);
- spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery, collector);
assertTrue("spans is null and it shouldn't be", spans != null);
- checkSpans(spans, 2, new int[]{8, 8});
+ checkSpans(spans, collector, 2, new int[]{8, 8});
closeIndexReader.close();
directory.close();
}
@@ -272,15 +275,17 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 1, true);
- Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+ Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, collector);
TopDocs topDocs = is.search(snq, 1);
Set<String> payloadSet = new HashSet<>();
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
- Collection<byte[]> payloads = spans.getPayload();
-
+ collector.reset();
+ spans.collect(collector);
+ Collection<byte[]> payloads = collector.getPayloads();
for (final byte [] payload : payloads) {
payloadSet.add(new String(payload, StandardCharsets.UTF_8));
}
@@ -310,14 +315,17 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 0, true);
- Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+ Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, collector);
TopDocs topDocs = is.search(snq, 1);
Set<String> payloadSet = new HashSet<>();
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
- Collection<byte[]> payloads = spans.getPayload();
+ collector.reset();
+ spans.collect(collector);
+ Collection<byte[]> payloads = collector.getPayloads();
for (final byte [] payload : payloads) {
payloadSet.add(new String(payload, StandardCharsets.UTF_8));
@@ -348,14 +356,17 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 0, true);
- Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
+ PayloadSpanCollector collector = new PayloadSpanCollector();
+ Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq, collector);
TopDocs topDocs = is.search(snq, 1);
Set<String> payloadSet = new HashSet<>();
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
- Collection<byte[]> payloads = spans.getPayload();
+ collector.reset();
+ spans.collect(collector);
+ Collection<byte[]> payloads = collector.getPayloads();
for (final byte [] payload : payloads) {
payloadSet.add(new String(payload, StandardCharsets.UTF_8));
@@ -401,7 +412,7 @@ public class TestPayloadSpans extends Lu
directory.close();
}
- private void checkSpans(Spans spans, int expectedNumSpans, int expectedNumPayloads,
+ private void checkSpans(Spans spans, PayloadSpanCollector collector, int expectedNumSpans, int expectedNumPayloads,
int expectedPayloadLength, int expectedFirstByte) throws IOException {
assertTrue("spans is null and it shouldn't be", spans != null);
//each position match should have a span associated with it, since there is just one underlying term query, there should
@@ -409,16 +420,16 @@ public class TestPayloadSpans extends Lu
int seen = 0;
while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
- assertEquals("isPayloadAvailable should return true/false as payloads are expected", expectedNumPayloads > 0, spans.isPayloadAvailable());
- //See payload helper, for the PayloadHelper.FIELD field, there is a single byte payload at every token
- if (spans.isPayloadAvailable()) {
- Collection<byte[]> payload = spans.getPayload();
- assertEquals("payload size", expectedNumPayloads, payload.size());
- for (final byte [] thePayload : payload) {
- assertEquals("payload length", expectedPayloadLength, thePayload.length);
- assertEquals("payload first byte", expectedFirstByte, thePayload[0]);
- }
+ collector.reset();
+ spans.collect(collector);
+
+ Collection<byte[]> payload = collector.getPayloads();
+ assertEquals("payload size", expectedNumPayloads, payload.size());
+ for (final byte [] thePayload : payload) {
+ assertEquals("payload length", expectedPayloadLength, thePayload.length);
+ assertEquals("payload first byte", expectedFirstByte, thePayload[0]);
}
+
seen++;
}
}
@@ -446,26 +457,26 @@ public class TestPayloadSpans extends Lu
return searcher;
}
- private void checkSpans(Spans spans, int numSpans, int[] numPayloads) throws IOException {
+ private void checkSpans(Spans spans, PayloadSpanCollector collector, int numSpans, int[] numPayloads) throws IOException {
int cnt = 0;
while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
if(VERBOSE)
System.out.println("\nSpans Dump --");
- if (spans.isPayloadAvailable()) {
- Collection<byte[]> payload = spans.getPayload();
- if(VERBOSE) {
- System.out.println("payloads for span:" + payload.size());
- for (final byte [] bytes : payload) {
- System.out.println("doc:" + spans.docID() + " s:" + spans.startPosition() + " e:" + spans.endPosition() + " "
+ collector.reset();
+ spans.collect(collector);
+
+ Collection<byte[]> payload = collector.getPayloads();
+ if(VERBOSE) {
+ System.out.println("payloads for span:" + payload.size());
+ for (final byte [] bytes : payload) {
+ System.out.println("doc:" + spans.docID() + " s:" + spans.startPosition() + " e:" + spans.endPosition() + " "
+ new String(bytes, StandardCharsets.UTF_8));
- }
}
- assertEquals("payload size", numPayloads[cnt], payload.size());
- } else { // no payload available
- assertFalse("Expected spans:" + numPayloads[cnt] + " found: 0", numPayloads.length > 0 && numPayloads[cnt] > 0 );
}
+ assertEquals("payload size", numPayloads[cnt], payload.size());
+
cnt++;
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java Tue May 19 09:21:24 2015
@@ -17,17 +17,16 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.Bits;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
/**
* Holds all implementations of classes in the o.a.l.s.spans package as a
* back-compatibility test. It does not run any tests per-se, however if
@@ -65,19 +64,14 @@ final class JustCompileSearchSpans {
public int endPosition() {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
-
- @Override
- public int nextStartPosition() throws IOException {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
@Override
- public Collection<byte[]> getPayload() {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+ public void collect(SpanCollector collector) throws IOException {
+
}
@Override
- public boolean isPayloadAvailable() {
+ public int nextStartPosition() throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@@ -100,7 +94,7 @@ final class JustCompileSearchSpans {
}
@Override
- public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts) {
+ public Spans getSpans(LeafReaderContext context, Bits acceptDocs, Map<Term,TermContext> termContexts, SpanCollector collector) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@@ -137,19 +131,14 @@ final class JustCompileSearchSpans {
public int endPosition() {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
-
- @Override
- public int nextStartPosition() throws IOException {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
@Override
- public Collection<byte[]> getPayload() {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+ public void collect(SpanCollector collector) throws IOException {
+
}
@Override
- public boolean isPayloadAvailable() {
+ public int nextStartPosition() throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java?rev=1680208&r1=1680207&r2=1680208&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java Tue May 19 09:21:24 2015
@@ -17,11 +17,6 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -30,6 +25,11 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
import org.apache.lucene.util.Bits;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
/**
*
* A wrapper to perform span operations on a non-leaf reader context
@@ -40,6 +40,10 @@ import org.apache.lucene.util.Bits;
public class MultiSpansWrapper {
public static Spans wrap(IndexReader reader, SpanQuery spanQuery) throws IOException {
+ return wrap(reader, spanQuery, SpanCollector.NO_OP);
+ }
+
+ public static Spans wrap(IndexReader reader, SpanQuery spanQuery, SpanCollector collector) throws IOException {
LeafReader lr = SlowCompositeReaderWrapper.wrap(reader); // slow, but ok for testing
LeafReaderContext lrContext = lr.getContext();
SpanQuery rewrittenQuery = (SpanQuery) spanQuery.rewrite(lr); // get the term contexts so getSpans can be called directly
@@ -50,7 +54,7 @@ public class MultiSpansWrapper {
TermContext termContext = TermContext.build(lrContext, term);
termContexts.put(term, termContext);
}
- Spans actSpans = spanQuery.getSpans(lrContext, new Bits.MatchAllBits(lr.numDocs()), termContexts);
+ Spans actSpans = spanQuery.getSpans(lrContext, new Bits.MatchAllBits(lr.numDocs()), termContexts, collector);
return actSpans;
}
}