You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2015/08/04 18:42:40 UTC
svn commit: r1694086 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/search/spans/
lucene/core/src/test/org/apache/lucene/search/spans/ lucene/test-framework/
lucene/test-framework/src/java/org/apache...
Author: romseygeek
Date: Tue Aug 4 16:42:40 2015
New Revision: 1694086
URL: http://svn.apache.org/r1694086
Log:
LUCENE-6580: Allow defined-width gaps in SpanNearQuery
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanNearQuery.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/SpanTestUtil.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1694086&r1=1694085&r2=1694086&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Tue Aug 4 16:42:40 2015
@@ -120,6 +120,9 @@ New Features
* LUCENE-6710: GeoPointField now uses full 64 bits (up from 62) to encode
lat/lon (Nick Knize via Mike McCandless).
+* LUCENE-6580: SpanNearQuery now allows defined-width gaps in its subqueries
+ (Alan Woodward, Adrien Grand).
+
API Changes
* LUCENE-6508: Simplify Lock api, there is now just
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java?rev=1694086&r1=1694085&r2=1694086&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/FilterSpans.java Tue Aug 4 16:42:40 2015
@@ -17,11 +17,11 @@ package org.apache.lucene.search.spans;
* limitations under the License.
*/
-import org.apache.lucene.search.TwoPhaseIterator;
-
import java.io.IOException;
import java.util.Objects;
+import org.apache.lucene.search.TwoPhaseIterator;
+
/**
* A {@link Spans} implementation wrapping another spans instance,
* allowing to filter spans matches easily by implementing {@link #accept}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java?rev=1694086&r1=1694085&r2=1694086&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java Tue Aug 4 16:42:40 2015
@@ -104,12 +104,9 @@ public class NearSpansOrdered extends Ne
Spans spans = subSpans[i];
assert spans.startPosition() != NO_MORE_POSITIONS;
assert spans.endPosition() != NO_MORE_POSITIONS;
-
- while (prevSpans.endPosition() > spans.startPosition()) { // while overlapping spans
- if (spans.nextStartPosition() == NO_MORE_POSITIONS) {
- oneExhaustedInCurrentDoc = true;
- return false;
- }
+ if (advancePosition(spans, prevSpans.endPosition()) == NO_MORE_POSITIONS) {
+ oneExhaustedInCurrentDoc = true;
+ return false;
}
matchWidth += (spans.startPosition() - prevSpans.endPosition());
prevSpans = spans;
@@ -118,6 +115,16 @@ public class NearSpansOrdered extends Ne
return true; // all subSpans ordered and non overlapping
}
+ private static int advancePosition(Spans spans, int position) throws IOException {
+ if (spans instanceof SpanNearQuery.GapSpans) {
+ return ((SpanNearQuery.GapSpans)spans).skipToPosition(position);
+ }
+ while (spans.startPosition() < position) {
+ spans.nextStartPosition();
+ }
+ return spans.startPosition();
+ }
+
@Override
public int startPosition() {
return atFirstInCurrentDoc ? -1 : matchStart;
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java?rev=1694086&r1=1694085&r2=1694086&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java Tue Aug 4 16:42:40 2015
@@ -20,8 +20,10 @@ package org.apache.lucene.search.spans;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import org.apache.lucene.index.IndexReader;
@@ -38,6 +40,77 @@ import org.apache.lucene.util.ToStringUt
* matches are required to be in-order.
*/
public class SpanNearQuery extends SpanQuery implements Cloneable {
+
+ /**
+ * A builder for SpanNearQueries
+ */
+ public static class Builder {
+ private final boolean ordered;
+ private final String field;
+ private final List<SpanQuery> clauses = new LinkedList<>();
+ private int slop;
+
+ /**
+ * Construct a new builder
+ * @param field the field to search in
+ * @param ordered whether or not clauses must be in-order to match
+ */
+ public Builder(String field, boolean ordered) {
+ this.field = field;
+ this.ordered = ordered;
+ }
+
+ /**
+ * Add a new clause
+ */
+ public Builder addClause(SpanQuery clause) {
+ if (Objects.equals(clause.getField(), field) == false)
+ throw new IllegalArgumentException("Cannot add clause " + clause + " to SpanNearQuery for field " + field);
+ this.clauses.add(clause);
+ return this;
+ }
+
+ /**
+ * Add a gap after the previous clause of a defined width
+ */
+ public Builder addGap(int width) {
+ if (!ordered)
+ throw new IllegalArgumentException("Gaps can only be added to ordered near queries");
+ this.clauses.add(new SpanGapQuery(field, width));
+ return this;
+ }
+
+ /**
+ * Set the slop for this query
+ */
+ public Builder setSlop(int slop) {
+ this.slop = slop;
+ return this;
+ }
+
+ /**
+ * Build the query
+ */
+ public SpanNearQuery build() {
+ return new SpanNearQuery(clauses.toArray(new SpanQuery[clauses.size()]), slop, ordered);
+ }
+
+ }
+
+ /**
+ * Returns a {@link Builder} for an ordered query on a particular field
+ */
+ public static Builder newOrderedNearQuery(String field) {
+ return new Builder(field, true);
+ }
+
+ /**
+ * Returns a {@link Builder} for an unordered query on a particular field
+ */
+ public static Builder newUnorderedNearQuery(String field) {
+ return new Builder(field, false);
+ }
+
protected List<SpanQuery> clauses;
protected int slop;
protected boolean inOrder;
@@ -221,4 +294,115 @@ public class SpanNearQuery extends SpanQ
int fac = 1 + (inOrder ? 8 : 4);
return fac * result;
}
+
+ private static class SpanGapQuery extends SpanQuery {
+
+ private final String field;
+ private final int width;
+
+ public SpanGapQuery(String field, int width) {
+ this.field = field;
+ this.width = width;
+ }
+
+ @Override
+ public String getField() {
+ return field;
+ }
+
+ @Override
+ public String toString(String field) {
+ return "SpanGap(" + field + ":" + width + ")";
+ }
+
+ @Override
+ public SpanWeight createWeight(IndexSearcher searcher, boolean needsScores) throws IOException {
+ return new SpanGapWeight(searcher);
+ }
+
+ private class SpanGapWeight extends SpanWeight {
+
+ SpanGapWeight(IndexSearcher searcher) throws IOException {
+ super(SpanGapQuery.this, searcher, null);
+ }
+
+ @Override
+ public void extractTermContexts(Map<Term, TermContext> contexts) {
+
+ }
+
+ @Override
+ public Spans getSpans(LeafReaderContext ctx, Postings requiredPostings) throws IOException {
+ return new GapSpans(width);
+ }
+
+ @Override
+ public void extractTerms(Set<Term> terms) {
+
+ }
+ }
+ }
+
+ static class GapSpans extends Spans {
+
+ int doc = -1;
+ int pos = -1;
+ final int width;
+
+ GapSpans(int width) {
+ this.width = width;
+ }
+
+ @Override
+ public int nextStartPosition() throws IOException {
+ return ++pos;
+ }
+
+ public int skipToPosition(int position) throws IOException {
+ return pos = position;
+ }
+
+ @Override
+ public int startPosition() {
+ return pos;
+ }
+
+ @Override
+ public int endPosition() {
+ return pos + width;
+ }
+
+ @Override
+ public int width() {
+ return width;
+ }
+
+ @Override
+ public void collect(SpanCollector collector) throws IOException {
+
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ pos = -1;
+ return ++doc;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ pos = -1;
+ return doc = target;
+ }
+
+ @Override
+ public long cost() {
+ return 0;
+ }
+ }
+
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java?rev=1694086&r1=1694085&r2=1694086&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/search/spans/Spans.java Tue Aug 4 16:42:40 2015
@@ -102,4 +102,5 @@ public abstract class Spans extends DocI
sb.append(")");
return sb.toString();
}
+
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java?rev=1694086&r1=1694085&r2=1694086&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java Tue Aug 4 16:42:40 2015
@@ -20,9 +20,9 @@ package org.apache.lucene.search.spans;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.CheckHits;
@@ -33,7 +33,8 @@ import org.apache.lucene.search.Weight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
-import static org.apache.lucene.search.spans.SpanTestUtil.*;
+import static org.apache.lucene.search.spans.SpanTestUtil.assertFinished;
+import static org.apache.lucene.search.spans.SpanTestUtil.assertNext;
public class TestNearSpansOrdered extends LuceneTestCase {
protected IndexSearcher searcher;
@@ -69,7 +70,8 @@ public class TestNearSpansOrdered extend
"w1 w3 w2 w3 zz",
"w1 xx w2 yy w3",
"w1 w3 xx w2 yy w3 zz",
- "t1 t2 t2 t1"
+ "t1 t2 t2 t1",
+ "g x x g g x x x g g x x g"
};
protected SpanNearQuery makeQuery(String s1, String s2, String s3,
@@ -240,7 +242,6 @@ public class TestNearSpansOrdered extend
assertFinished(spans);
}
-
/**
* not a direct test of NearSpans, but a demonstration of how/when
* this causes problems
@@ -252,4 +253,52 @@ public class TestNearSpansOrdered extend
+ e.toString(),
0.0f < e.getValue());
}
+
+ public void testGaps() throws Exception {
+ SpanNearQuery q = SpanNearQuery.newOrderedNearQuery(FIELD)
+ .addClause(new SpanTermQuery(new Term(FIELD, "w1")))
+ .addGap(1)
+ .addClause(new SpanTermQuery(new Term(FIELD, "w2")))
+ .build();
+ Spans spans = MultiSpansWrapper.wrap(reader, q);
+ assertNext(spans, 1, 0, 3);
+ assertNext(spans, 2, 0, 3);
+ assertFinished(spans);
+
+ q = SpanNearQuery.newOrderedNearQuery(FIELD)
+ .addClause(new SpanTermQuery(new Term(FIELD, "w1")))
+ .addGap(1)
+ .addClause(new SpanTermQuery(new Term(FIELD, "w2")))
+ .addGap(1)
+ .addClause(new SpanTermQuery(new Term(FIELD, "w3")))
+ .setSlop(1)
+ .build();
+ spans = MultiSpansWrapper.wrap(reader, q);
+ assertNext(spans, 2, 0, 5);
+ assertNext(spans, 3, 0, 6);
+ assertFinished(spans);
+ }
+
+ public void testMultipleGaps() throws Exception {
+ SpanQuery q = SpanNearQuery.newOrderedNearQuery(FIELD)
+ .addClause(new SpanTermQuery(new Term(FIELD, "g")))
+ .addGap(2)
+ .addClause(new SpanTermQuery(new Term(FIELD, "g")))
+ .build();
+ Spans spans = MultiSpansWrapper.wrap(reader, q);
+ assertNext(spans, 5, 0, 4);
+ assertNext(spans, 5, 9, 13);
+ assertFinished(spans);
+ }
+
+ /*
+ protected String[] docFields = {
+ "w1 w2 w3 w4 w5",
+ "w1 w3 w2 w3 zz",
+ "w1 xx w2 yy w3",
+ "w1 w3 xx w2 yy w3 zz",
+ "t1 t2 t2 t1",
+ "g x x g g x x x g g x x g"
+ };
+ */
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanNearQuery.java?rev=1694086&r1=1694085&r2=1694086&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanNearQuery.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanNearQuery.java Tue Aug 4 16:42:40 2015
@@ -79,4 +79,22 @@ public class TestSpanNearQuery extends L
ir.close();
dir.close();
}
+
+ public void testBuilder() throws Exception {
+
+ // Can't add subclauses from different fields
+ try {
+ SpanNearQuery.newOrderedNearQuery("field1").addClause(new SpanTermQuery(new Term("field2", "term")));
+ fail("Expected an error when adding a clause with a different field");
+ }
+ catch (IllegalArgumentException e) {}
+
+ // Can't add gaps to unordered queries
+ try {
+ SpanNearQuery.newUnorderedNearQuery("field1").addGap(1);
+ fail("Expected an error when adding a gap to an unordered query");
+ }
+ catch (IllegalArgumentException e) {}
+
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/SpanTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/SpanTestUtil.java?rev=1694086&r1=1694085&r2=1694086&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/SpanTestUtil.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/search/spans/SpanTestUtil.java Tue Aug 4 16:42:40 2015
@@ -125,11 +125,12 @@ public class SpanTestUtil {
* Makes a new unordered SpanNearQuery (with additional asserts) from the provided {@code terms}
*/
public static SpanQuery spanNearUnorderedQuery(String field, int slop, String... terms) {
- SpanQuery[] subqueries = new SpanQuery[terms.length];
- for (int i = 0; i < terms.length; i++) {
- subqueries[i] = spanTermQuery(field, terms[i]);
+ SpanNearQuery.Builder builder = SpanNearQuery.newUnorderedNearQuery(field);
+ builder.setSlop(slop);
+ for (String term : terms) {
+ builder.addClause(new SpanTermQuery(new Term(field, term)));
}
- return spanNearUnorderedQuery(slop, subqueries);
+ return spanQuery(builder.build());
}
/**