You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2015/08/03 17:49:26 UTC
svn commit: r1693921 - in /lucene/dev/trunk/lucene: ./
core/src/java/org/apache/lucene/search/payloads/
core/src/java/org/apache/lucene/search/spans/
core/src/test/org/apache/lucene/search/payloads/
core/src/test/org/apache/lucene/search/spans/
Author: romseygeek
Date: Mon Aug 3 15:49:25 2015
New Revision: 1693921
URL: http://svn.apache.org/r1693921
Log:
LUCENE-6706: Add PayloadScoreQuery
Added:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadScoreQuery.java (with props)
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadScoreQuery.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1693921&r1=1693920&r2=1693921&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon Aug 3 15:49:25 2015
@@ -141,6 +141,9 @@ New Features
* LUCENE-6695: Added a new BlendedTermQuery to blend statistics across several
terms. (Simon Willnauer, Adrien Grand)
+* LUCENE-6706: Added a new PayloadScoreQuery that generalises the behaviour of
+ PayloadTermQuery and PayloadNearQuery to all Span queries. (Alan Woodward)
+
* LUCENE-6697: Add experimental range tree doc values format and
queries, based on a 1D version of the spatial BKD tree, for a faster
and smaller alternative to postings-based numeric and binary term
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1693921&r1=1693920&r2=1693921&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Mon Aug 3 15:49:25 2015
@@ -17,6 +17,14 @@ package org.apache.lucene.search.payload
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
@@ -31,18 +39,9 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.search.spans.SpanScorer;
import org.apache.lucene.search.spans.SpanWeight;
import org.apache.lucene.search.spans.Spans;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ToStringUtils;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
/**
* This class is very similar to
* {@link org.apache.lucene.search.spans.SpanNearQuery} except that it factors
@@ -56,6 +55,8 @@ import java.util.Objects;
* Payload scores are aggregated using a pluggable {@link PayloadFunction}.
*
* @see org.apache.lucene.search.similarities.Similarity.SimScorer#computePayloadFactor(int, int, int, BytesRef)
+ *
+ * @deprecated use {@link PayloadScoreQuery} to wrap {@link SpanNearQuery}
*/
public class PayloadNearQuery extends SpanNearQuery {
@@ -215,22 +216,17 @@ public class PayloadNearQuery extends Sp
}
}
- //
@Override
- protected void setFreqCurrentDoc() throws IOException {
- freq = 0.0f;
+ protected void doStartCurrentDoc() throws IOException {
payloadScore = 0;
payloadsSeen = 0;
- int startPos = spans.nextStartPosition();
- assert startPos != Spans.NO_MORE_POSITIONS : "initial startPos NO_MORE_POSITIONS, spans="+spans;
- do {
- int matchLength = spans.endPosition() - startPos;
- freq += docScorer.computeSlopFactor(matchLength);
- collector.reset();
- spans.collect(collector);
- processPayloads(collector.getPayloads(), startPos, spans.endPosition());
- startPos = spans.nextStartPosition();
- } while (startPos != Spans.NO_MORE_POSITIONS);
+ }
+
+ @Override
+ protected void doCurrentSpans() throws IOException {
+ collector.reset();
+ spans.collect(collector);
+ processPayloads(collector.getPayloads(), spans.startPosition(), spans.endPosition());
}
@Override
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadScoreQuery.java?rev=1693921&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadScoreQuery.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadScoreQuery.java Mon Aug 3 15:49:25 2015
@@ -0,0 +1,215 @@
+package org.apache.lucene.search.payloads;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.spans.SpanCollector;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanScorer;
+import org.apache.lucene.search.spans.SpanWeight;
+import org.apache.lucene.search.spans.Spans;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * A Query class that uses a {@link PayloadFunction} to modify the score of a
+ * wrapped SpanQuery
+ *
+ * NOTE: In order to take advantage of this with the default scoring implementation
+ * ({@link DefaultSimilarity}), you must override {@link DefaultSimilarity#scorePayload(int, int, int, BytesRef)},
+ * which returns 1 by default.
+ *
+ * @see org.apache.lucene.search.similarities.Similarity.SimScorer#computePayloadFactor(int, int, int, BytesRef)
+ */
+public class PayloadScoreQuery extends SpanQuery {
+
+ private final SpanQuery wrappedQuery;
+ private final PayloadFunction function;
+
+ /**
+ * Creates a new PayloadScoreQuery
+ * @param wrappedQuery the query to wrap
+ * @param function a PayloadFunction to use to modify the scores
+ */
+ public PayloadScoreQuery(SpanQuery wrappedQuery, PayloadFunction function) {
+ this.wrappedQuery = wrappedQuery;
+ this.function = function;
+ }
+
+ @Override
+ public String getField() {
+ return wrappedQuery.getField();
+ }
+
+ @Override
+ public String toString(String field) {
+ return "PayloadSpanQuery[" + wrappedQuery.toString(field) + "; " + function.toString() + "]";
+ }
+
+ @Override
+ public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+ SpanWeight innerWeight = wrappedQuery.createWeight(searcher, needsScores);
+ if (!needsScores)
+ return innerWeight;
+ return new PayloadSpanWeight(searcher, innerWeight);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof PayloadScoreQuery)) return false;
+ if (!super.equals(o)) return false;
+
+ PayloadScoreQuery that = (PayloadScoreQuery) o;
+
+ if (wrappedQuery != null ? !wrappedQuery.equals(that.wrappedQuery) : that.wrappedQuery != null) return false;
+ return !(function != null ? !function.equals(that.function) : that.function != null);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + (wrappedQuery != null ? wrappedQuery.hashCode() : 0);
+ result = 31 * result + (function != null ? function.hashCode() : 0);
+ return result;
+ }
+
+ private class PayloadSpanWeight extends SpanWeight {
+
+ private final SpanWeight innerWeight;
+
+ public PayloadSpanWeight(IndexSearcher searcher, SpanWeight innerWeight) throws IOException {
+ super(PayloadScoreQuery.this, searcher, null);
+ this.innerWeight = innerWeight;
+ }
+
+ @Override
+ public void extractTermContexts(Map<Term, TermContext> contexts) {
+ innerWeight.extractTermContexts(contexts);
+ }
+
+ @Override
+ public Spans getSpans(LeafReaderContext ctx, Postings requiredPostings) throws IOException {
+ return innerWeight.getSpans(ctx, requiredPostings.atLeast(Postings.PAYLOADS));
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ Spans spans = getSpans(context, Postings.PAYLOADS);
+ if (spans == null)
+ return null;
+ return new PayloadSpanScorer(spans, this, innerWeight.getSimScorer(context));
+ }
+
+ @Override
+ public void extractTerms(Set<Term> terms) {
+ innerWeight.extractTerms(terms);
+ }
+
+ @Override
+ public float getValueForNormalization() throws IOException {
+ return innerWeight.getValueForNormalization();
+ }
+
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ innerWeight.normalize(queryNorm, topLevelBoost);
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ PayloadSpanScorer scorer = (PayloadSpanScorer) scorer(context);
+ if (scorer == null || scorer.advance(doc) != doc)
+ return Explanation.noMatch("No match");
+
+ SpanWeight innerWeight = ((PayloadSpanWeight)scorer.getWeight()).innerWeight;
+ Explanation innerExpl = innerWeight.explain(context, doc);
+ scorer.freq(); // force freq calculation
+ Explanation payloadExpl = scorer.getPayloadExplanation();
+
+ return Explanation.match(scorer.scoreCurrentDoc(), "PayloadSpanQuery, product of:", innerExpl, payloadExpl);
+ }
+ }
+
+ private class PayloadSpanScorer extends SpanScorer implements SpanCollector {
+
+ private int payloadsSeen;
+ private float payloadScore;
+
+ private PayloadSpanScorer(Spans spans, SpanWeight weight, Similarity.SimScorer docScorer) throws IOException {
+ super(spans, weight, docScorer);
+ }
+
+ @Override
+ protected void doStartCurrentDoc() {
+ payloadScore = 0;
+ payloadsSeen = 0;
+ }
+
+ @Override
+ protected void doCurrentSpans() throws IOException {
+ spans.collect(this);
+ }
+
+ @Override
+ public void collectLeaf(PostingsEnum postings, int position, Term term) throws IOException {
+ BytesRef payload = postings.getPayload();
+ if (payload == null)
+ return;
+ float payloadFactor = docScorer.computePayloadFactor(docID(), spans.startPosition(), spans.endPosition(), payload);
+ payloadScore = function.currentScore(docID(), getField(), spans.startPosition(), spans.endPosition(),
+ payloadsSeen, payloadScore, payloadFactor);
+ payloadsSeen++;
+ }
+
+ protected float getPayloadScore() {
+ return function.docScore(docID(), getField(), payloadsSeen, payloadScore);
+ }
+
+ protected Explanation getPayloadExplanation() {
+ return function.explain(docID(), getField(), payloadsSeen, payloadScore);
+ }
+
+ protected float getSpanScore() throws IOException {
+ return super.scoreCurrentDoc();
+ }
+
+ @Override
+ protected float scoreCurrentDoc() throws IOException {
+ return getSpanScore() * getPayloadScore();
+ }
+
+ @Override
+ public void reset() {
+
+ }
+ }
+
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1693921&r1=1693920&r2=1693921&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Mon Aug 3 15:49:25 2015
@@ -17,6 +17,11 @@ package org.apache.lucene.search.payload
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Objects;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
@@ -34,11 +39,6 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.util.BytesRef;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Objects;
-
/**
* This class is very similar to
* {@link org.apache.lucene.search.spans.SpanTermQuery} except that it factors
@@ -51,6 +51,8 @@ import java.util.Objects;
* <p>
* Payload scores are aggregated using a pluggable {@link PayloadFunction}.
* @see org.apache.lucene.search.similarities.Similarity.SimScorer#computePayloadFactor(int, int, int, BytesRef)
+ *
+ * @deprecated use {@link PayloadScoreQuery} to wrap {@link SpanTermQuery}
**/
public class PayloadTermQuery extends SpanTermQuery {
protected PayloadFunction function;
@@ -114,27 +116,16 @@ public class PayloadTermQuery extends Sp
}
@Override
- protected void setFreqCurrentDoc() throws IOException {
- freq = 0.0f;
- numMatches = 0;
+ protected void doStartCurrentDoc() throws IOException {
payloadScore = 0;
payloadsSeen = 0;
- int startPos = spans.nextStartPosition();
- assert startPos != Spans.NO_MORE_POSITIONS : "initial startPos NO_MORE_POSITIONS, spans="+spans;
- do {
- int matchLength = spans.endPosition() - startPos;
- if (docScorer == null) {
- freq = 1;
- return;
- }
- freq += docScorer.computeSlopFactor(matchLength);
- numMatches++;
- payloadCollector.reset();
- spans.collect(payloadCollector);
- processPayload();
+ }
- startPos = spans.nextStartPosition();
- } while (startPos != Spans.NO_MORE_POSITIONS);
+ @Override
+ protected void doCurrentSpans() throws IOException {
+ payloadCollector.reset();
+ spans.collect(payloadCollector);
+ processPayload();
}
protected void processPayload() throws IOException {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java?rev=1693921&r1=1693920&r2=1693921&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/ContainSpans.java Mon Aug 3 15:49:25 2015
@@ -54,7 +54,8 @@ abstract class ContainSpans extends Conj
@Override
public void collect(SpanCollector collector) throws IOException {
- sourceSpans.collect(collector);
+ bigSpans.collect(collector);
+ littleSpans.collect(collector);
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java?rev=1693921&r1=1693920&r2=1693921&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java Mon Aug 3 15:49:25 2015
@@ -17,13 +17,13 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Objects;
+
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.similarities.Similarity;
-import java.io.IOException;
-import java.util.Objects;
-
/**
* Public for extension only.
*/
@@ -76,10 +76,12 @@ public class SpanScorer extends Scorer {
* <p>
* This will be called at most once per document.
*/
- protected void setFreqCurrentDoc() throws IOException {
+ protected final void setFreqCurrentDoc() throws IOException {
freq = 0.0f;
numMatches = 0;
+ doStartCurrentDoc();
+
assert spans.startPosition() == -1 : "incorrect initial start position, spans="+spans;
assert spans.endPosition() == -1 : "incorrect initial end position, spans="+spans;
int prevStartPos = -1;
@@ -100,6 +102,7 @@ public class SpanScorer extends Scorer {
return;
}
freq += docScorer.computeSlopFactor(spans.width());
+ doCurrentSpans();
prevStartPos = startPos;
prevEndPos = endPos;
startPos = spans.nextStartPosition();
@@ -108,6 +111,16 @@ public class SpanScorer extends Scorer {
assert spans.startPosition() == Spans.NO_MORE_POSITIONS : "incorrect final start position, spans="+spans;
assert spans.endPosition() == Spans.NO_MORE_POSITIONS : "incorrect final end position, spans="+spans;
}
+
+ /**
+ * Called before the current doc's frequency is calculated
+ */
+ protected void doStartCurrentDoc() throws IOException {}
+
+ /**
+ * Called each time the scorer's Spans is advanced during frequency calculation
+ */
+ protected void doCurrentSpans() throws IOException {}
/**
* Score the current doc. The default implementation scores the doc
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java?rev=1693921&r1=1693920&r2=1693921&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java Mon Aug 3 15:49:25 2015
@@ -17,6 +17,9 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Map;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
@@ -30,10 +33,6 @@ import org.apache.lucene.search.TermStat
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
-import org.apache.lucene.util.Bits;
-
-import java.io.IOException;
-import java.util.Map;
/**
* Expert-only. Public for use by other weight implementations
@@ -141,10 +140,14 @@ public abstract class SpanWeight extends
throw new IllegalStateException("field \"" + field + "\" was indexed without position data; cannot run SpanQuery (query=" + parentQuery + ")");
}
Spans spans = getSpans(context, Postings.POSITIONS);
- Similarity.SimScorer simScorer = simWeight == null ? null : similarity.simScorer(simWeight, context);
+ Similarity.SimScorer simScorer = getSimScorer(context);
return (spans == null) ? null : new SpanScorer(spans, this, simScorer);
}
+ public Similarity.SimScorer getSimScorer(LeafReaderContext context) throws IOException {
+ return simWeight == null ? null : similarity.simScorer(simWeight, context);
+ }
+
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
SpanScorer scorer = (SpanScorer) scorer(context);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java?rev=1693921&r1=1693920&r2=1693921&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java Mon Aug 3 15:49:25 2015
@@ -18,17 +18,20 @@ package org.apache.lucene.search.payload
*/
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.BaseExplanationTestCase;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.util.BytesRef;
/**
* TestExplanations subclass focusing on payload queries
*/
public class TestPayloadExplanations extends BaseExplanationTestCase {
- private PayloadFunction functions[] = new PayloadFunction[] {
+
+ private static PayloadFunction functions[] = new PayloadFunction[] {
new AveragePayloadFunction(),
new MinPayloadFunction(),
new MaxPayloadFunction(),
@@ -89,4 +92,45 @@ public class TestPayloadExplanations ext
}
// TODO: test the payloadnear query too!
+
+ /*
+ protected static final String[] docFields = {
+ "w1 w2 w3 w4 w5",
+ "w1 w3 w2 w3 zz",
+ "w1 xx w2 yy w3",
+ "w1 w3 xx w2 yy w3 zz"
+ };
+ */
+
+ public void testAllFunctions(SpanQuery query, int[] expected) throws Exception {
+ for (PayloadFunction fn : functions) {
+ qtest(new PayloadScoreQuery(query, fn), expected);
+ }
+ }
+
+ public void testSimpleTerm() throws Exception {
+ SpanTermQuery q = new SpanTermQuery(new Term(FIELD, "w2"));
+ testAllFunctions(q, new int[]{ 0, 1, 2, 3});
+ }
+
+ public void testOrTerm() throws Exception {
+ SpanOrQuery q = new SpanOrQuery(
+ new SpanTermQuery(new Term(FIELD, "xx")), new SpanTermQuery(new Term(FIELD, "yy"))
+ );
+ testAllFunctions(q, new int[]{ 2, 3 });
+ }
+
+ public void testOrderedNearQuery() throws Exception {
+ SpanNearQuery q = new SpanNearQuery(new SpanQuery[]{
+ new SpanTermQuery(new Term(FIELD, "w3")), new SpanTermQuery(new Term(FIELD, "w2"))
+ }, 1, true);
+ testAllFunctions(q, new int[]{ 1, 3 });
+ }
+
+ public void testUnorderedNearQuery() throws Exception {
+ SpanNearQuery q = new SpanNearQuery(new SpanQuery[]{
+ new SpanTermQuery(new Term(FIELD, "w2")), new SpanTermQuery(new Term(FIELD, "w3"))
+ }, 1, false);
+ testAllFunctions(q, new int[]{ 0, 1, 2, 3 });
+ }
}
Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadScoreQuery.java?rev=1693921&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadScoreQuery.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadScoreQuery.java Mon Aug 3 15:49:25 2015
@@ -0,0 +1,284 @@
+package org.apache.lucene.search.payloads;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.spans.SpanContainingQuery;
+import org.apache.lucene.search.spans.SpanNearQuery;
+import org.apache.lucene.search.spans.SpanOrQuery;
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.English;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestPayloadScoreQuery extends LuceneTestCase {
+
+ private static void checkQuery(SpanQuery query, PayloadFunction function, int[] expectedDocs, float[] expectedScores) throws IOException {
+
+ assertTrue("Expected docs and scores arrays must be the same length!", expectedDocs.length == expectedScores.length);
+
+ PayloadScoreQuery psq = new PayloadScoreQuery(query, function);
+ TopDocs hits = searcher.search(psq, expectedDocs.length);
+
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ if (i > expectedDocs.length - 1)
+ fail("Unexpected hit in document " + hits.scoreDocs[i].doc);
+ if (hits.scoreDocs[i].doc != expectedDocs[i])
+ fail("Unexpected hit in document " + hits.scoreDocs[i].doc);
+ assertEquals("Bad score in document " + expectedDocs[i], expectedScores[i], hits.scoreDocs[i].score, 0.000001);
+ }
+
+ if (hits.scoreDocs.length > expectedDocs.length)
+ fail("Unexpected hit in document " + hits.scoreDocs[expectedDocs.length]);
+ }
+
+ @Test
+ public void testTermQuery() throws IOException {
+
+ SpanTermQuery q = new SpanTermQuery(new Term("field", "eighteen"));
+ for (PayloadFunction fn
+ : new PayloadFunction[]{ new AveragePayloadFunction(), new MaxPayloadFunction(), new MinPayloadFunction() }) {
+ checkQuery(q, fn, new int[]{ 118, 218, 18 },
+ new float[] { 4.0f, 4.0f, 2.0f });
+ }
+
+ }
+
+ @Test
+ public void testOrQuery() throws IOException {
+
+ SpanOrQuery q = new SpanOrQuery(new SpanTermQuery(new Term("field", "eighteen")),
+ new SpanTermQuery(new Term("field", "nineteen")));
+ for (PayloadFunction fn
+ : new PayloadFunction[]{ new AveragePayloadFunction(), new MaxPayloadFunction(), new MinPayloadFunction() }) {
+ checkQuery(q, fn, new int[]{ 118, 119, 218, 219, 18, 19 },
+ new float[] { 4.0f, 4.0f, 4.0f, 4.0f, 2.0f, 2.0f });
+ }
+
+ }
+
+ @Test
+ public void testNearQuery() throws IOException {
+
+ // 2 4
+ // twenty two
+ // 2 4 4 4
+ // one hundred twenty two
+
+ SpanNearQuery q = new SpanNearQuery(new SpanQuery[]{
+ new SpanTermQuery(new Term("field", "twenty")),
+ new SpanTermQuery(new Term("field", "two"))
+ }, 0, true);
+
+ checkQuery(q, new MaxPayloadFunction(), new int[]{ 22, 122, 222 }, new float[]{ 4.0f, 4.0f, 4.0f });
+ checkQuery(q, new MinPayloadFunction(), new int[]{ 122, 222, 22 }, new float[]{ 4.0f, 4.0f, 2.0f });
+ checkQuery(q, new AveragePayloadFunction(), new int[] { 122, 222, 22 }, new float[] { 4.0f, 4.0f, 3.0f });
+
+ }
+
+ @Test
+ public void testNestedNearQuery() throws Exception {
+
+ // (one OR hundred) NEAR (twenty two) ~ 1
+ // 2 4 4 4
+ // one hundred twenty two
+ // two hundred twenty two
+
+ SpanNearQuery q = new SpanNearQuery(new SpanQuery[]{
+ new SpanOrQuery(new SpanTermQuery(new Term("field", "one")), new SpanTermQuery(new Term("field", "hundred"))),
+ new SpanNearQuery(new SpanQuery[]{
+ new SpanTermQuery(new Term("field", "twenty")),
+ new SpanTermQuery(new Term("field", "two"))
+ }, 0, true)
+ }, 1, true);
+
+ checkQuery(q, new MaxPayloadFunction(), new int[]{ 122, 222 }, new float[]{ 4.0f, 4.0f });
+ checkQuery(q, new MinPayloadFunction(), new int[]{ 222, 122 }, new float[]{ 4.0f, 2.0f });
+ checkQuery(q, new AveragePayloadFunction(), new int[] { 222, 122 }, new float[]{ 4.0f, 3.666666f });
+
+ }
+
+ @Test
+ public void testSpanContainingQuery() throws Exception {
+
+ // twenty WITHIN ((one OR hundred) NEAR two)~2
+ SpanContainingQuery q = new SpanContainingQuery(
+ new SpanNearQuery(new SpanQuery[]{
+ new SpanOrQuery(new SpanTermQuery(new Term("field", "one")), new SpanTermQuery(new Term("field", "hundred"))),
+ new SpanTermQuery(new Term("field", "two"))
+ }, 2, true),
+ new SpanTermQuery(new Term("field", "twenty"))
+ );
+
+ checkQuery(q, new AveragePayloadFunction(), new int[] { 222, 122 }, new float[]{ 4.0f, 3.666666f });
+ checkQuery(q, new MaxPayloadFunction(), new int[]{ 122, 222 }, new float[]{ 4.0f, 4.0f });
+ checkQuery(q, new MinPayloadFunction(), new int[]{ 222, 122 }, new float[]{ 4.0f, 2.0f });
+
+ }
+
+ private static IndexSearcher searcher;
+ private static IndexReader reader;
+ private static Directory directory;
+ private static BoostingSimilarity similarity = new BoostingSimilarity();
+ private static byte[] payload2 = new byte[]{2};
+ private static byte[] payload4 = new byte[]{4};
+
+ private static class PayloadAnalyzer extends Analyzer {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(result, new PayloadFilter(result));
+ }
+ }
+
+ private static class PayloadFilter extends TokenFilter {
+
+ private int numSeen = 0;
+ private final PayloadAttribute payAtt;
+
+ public PayloadFilter(TokenStream input) {
+ super(input);
+ payAtt = addAttribute(PayloadAttribute.class);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ boolean result = false;
+ if (input.incrementToken()) {
+ if (numSeen % 4 == 0) {
+ payAtt.setPayload(new BytesRef(payload2));
+ } else {
+ payAtt.setPayload(new BytesRef(payload4));
+ }
+ numSeen++;
+ result = true;
+ }
+ return result;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ this.numSeen = 0;
+ }
+ }
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
+ newIndexWriterConfig(new PayloadAnalyzer())
+ .setSimilarity(similarity));
+ //writer.infoStream = System.out;
+ for (int i = 0; i < 300; i++) {
+ Document doc = new Document();
+ doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES));
+ String txt = English.intToEnglish(i) +' '+English.intToEnglish(i+1);
+ doc.add(newTextField("field2", txt, Field.Store.YES));
+ writer.addDocument(doc);
+ }
+ reader = writer.getReader();
+ writer.close();
+
+ searcher = newSearcher(reader);
+ searcher.setSimilarity(similarity);
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ searcher = null;
+ reader.close();
+ reader = null;
+ directory.close();
+ directory = null;
+ }
+
+ static class BoostingSimilarity extends DefaultSimilarity {
+
+ @Override
+ public float queryNorm(float sumOfSquaredWeights) {
+ return 1.0f;
+ }
+
+ @Override
+ public float coord(int overlap, int maxOverlap) {
+ return 1.0f;
+ }
+
+ @Override
+ public float scorePayload(int docId, int start, int end, BytesRef payload) {
+ //we know it is size 4 here, so ignore the offset/length
+ return payload.bytes[payload.offset];
+ }
+
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ //Make everything else 1 so we see the effect of the payload
+ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ @Override
+ public float lengthNorm(FieldInvertState state) {
+ return state.getBoost();
+ }
+
+ @Override
+ public float sloppyFreq(int distance) {
+ return 1.0f;
+ }
+
+ @Override
+ public float tf(float freq) {
+ return 1.0f;
+ }
+
+ // idf used for phrase queries
+ @Override
+ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats) {
+ return Explanation.match(1.0f, "Inexplicable");
+ }
+
+ @Override
+ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
+ return Explanation.match(1.0f, "Inexplicable");
+ }
+
+ }
+
+}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1693921&r1=1693920&r2=1693921&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java Mon Aug 3 15:49:25 2015
@@ -17,11 +17,11 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.similarities.Similarity;
-import java.io.IOException;
-
/**
* Holds all implementations of classes in the o.a.l.s.spans package as a
* back-compatibility test. It does not run any tests per-se, however if
@@ -157,11 +157,6 @@ final class JustCompileSearchSpans {
}
@Override
- protected void setFreqCurrentDoc() {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
-
- @Override
protected float scoreCurrentDoc() throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}