You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2015/04/03 22:13:07 UTC
svn commit: r1671151 [1/2] - in /lucene/dev/branches/lucene6271: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/java/org/apache/lucene/search/payloads/
lucene/core/src/java/org/apache/lucene/search/spans/ lucene/c...
Author: rmuir
Date: Fri Apr 3 20:13:06 2015
New Revision: 1671151
URL: http://svn.apache.org/r1671151
Log:
merge trunk up to r1671137
Added:
lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/highlight/MissesTest.java
- copied unchanged from r1671143, lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/MissesTest.java
lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
- copied unchanged from r1671143, lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/BaseGlobalOrdinalScorer.java
lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
- copied unchanged from r1671143, lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsCollector.java
lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
- copied unchanged from r1671143, lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsQuery.java
lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java
- copied unchanged from r1671143, lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreCollector.java
lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
- copied unchanged from r1671143, lucene/dev/trunk/lucene/join/src/java/org/apache/lucene/search/join/GlobalOrdinalsWithScoreQuery.java
lucene/dev/branches/lucene6271/lucene/licenses/httpclient-4.4.1.jar.sha1
- copied unchanged from r1671143, lucene/dev/trunk/lucene/licenses/httpclient-4.4.1.jar.sha1
lucene/dev/branches/lucene6271/lucene/licenses/httpcore-4.4.1.jar.sha1
- copied unchanged from r1671143, lucene/dev/trunk/lucene/licenses/httpcore-4.4.1.jar.sha1
lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/OldBackupDirectory.java
- copied unchanged from r1671143, lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/OldBackupDirectory.java
lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
- copied unchanged from r1671143, lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/RestoreCore.java
lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java
- copied unchanged from r1671143, lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java
lucene/dev/branches/lucene6271/solr/licenses/httpclient-4.4.1.jar.sha1
- copied unchanged from r1671143, lucene/dev/trunk/solr/licenses/httpclient-4.4.1.jar.sha1
lucene/dev/branches/lucene6271/solr/licenses/httpcore-4.4.1.jar.sha1
- copied unchanged from r1671143, lucene/dev/trunk/solr/licenses/httpcore-4.4.1.jar.sha1
lucene/dev/branches/lucene6271/solr/licenses/httpmime-4.4.1.jar.sha1
- copied unchanged from r1671143, lucene/dev/trunk/solr/licenses/httpmime-4.4.1.jar.sha1
Removed:
lucene/dev/branches/lucene6271/lucene/licenses/httpclient-4.3.1.jar.sha1
lucene/dev/branches/lucene6271/lucene/licenses/httpcore-4.3.jar.sha1
lucene/dev/branches/lucene6271/solr/licenses/httpclient-4.3.1.jar.sha1
lucene/dev/branches/lucene6271/solr/licenses/httpcore-4.3.jar.sha1
lucene/dev/branches/lucene6271/solr/licenses/httpmime-4.3.1.jar.sha1
Modified:
lucene/dev/branches/lucene6271/ (props changed)
lucene/dev/branches/lucene6271/lucene/ (props changed)
lucene/dev/branches/lucene6271/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6271/lucene/core/ (props changed)
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpans.java
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansPayloadOrdered.java
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java
lucene/dev/branches/lucene6271/lucene/highlighter/ (props changed)
lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
lucene/dev/branches/lucene6271/lucene/ivy-versions.properties (contents, props changed)
lucene/dev/branches/lucene6271/lucene/join/ (props changed)
lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
lucene/dev/branches/lucene6271/lucene/licenses/ (props changed)
lucene/dev/branches/lucene6271/lucene/suggest/ (props changed)
lucene/dev/branches/lucene6271/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/SuggestFieldTest.java
lucene/dev/branches/lucene6271/lucene/test-framework/ (props changed)
lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java
lucene/dev/branches/lucene6271/solr/ (props changed)
lucene/dev/branches/lucene6271/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/lucene6271/solr/core/ (props changed)
lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java
lucene/dev/branches/lucene6271/solr/licenses/ (props changed)
Modified: lucene/dev/branches/lucene6271/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/CHANGES.txt?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6271/lucene/CHANGES.txt Fri Apr 3 20:13:06 2015
@@ -40,10 +40,15 @@ API Changes
New Features
-* LUCENE-6308: Span queries now share document conjunction/intersection
+* LUCENE-6308, LUCENE-6385, LUCENE-6391: Span queries now share
+ document conjunction/intersection
code with boolean queries, and use two-phased iterators for
faster intersection by avoiding loading positions in certain cases.
- (Paul Elschot, Robert Muir via Mike McCandless)
+ (Paul Elschot, Terry Smith, Robert Muir via Mike McCandless)
+
+* LUCENE-6352: Added a new query time join to the join module that uses
+ global ordinals, which is faster for subsequent joins between reopens.
+ (Martijn van Groningen, Adrien Grand)
Optimizations
@@ -52,6 +57,9 @@ Optimizations
faster IndexWriter.deleteAll in that case (Robert Muir, Adrien
Grand, Mike McCandless)
+* LUCENE-6388: Optimize SpanNearQuery when payloads are not present.
+ (Robert Muir)
+
Bug Fixes
* LUCENE-6378: Fix all RuntimeExceptions to throw the underlying root cause.
@@ -123,6 +131,10 @@ Bug Fixes
DocumentsWriterStallControl to prevent hangs during indexing if we
miss a .notify/All somewhere (Mike McCandless)
+* LUCENE-6386: Correct IndexWriter.forceMerge documentation to state
+ that up to 3X (X = current index size) spare disk space may be needed
+ to complete forceMerge(1). (Robert Muir, Shai Erera, Mike McCandless)
+
Optimizations
* LUCENE-6183, LUCENE-5647: Avoid recompressing stored fields
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Apr 3 20:13:06 2015
@@ -1547,14 +1547,15 @@ public class IndexWriter implements Clos
* longer be changed).</p>
*
* <p>Note that this requires free space that is proportional
- * to the size of the index in your Directory (2X if you're
- * using compound file format). For example, if your index
- * size is 10 MB then you need an additional 10 MB free for
- * this to complete (20 MB if you're using compound file
- * format). This is also affected by the {@link Codec} that
- * is used to execute the merge, and may result in even a
- * bigger index. Also, it's best to call {@link #commit()}
- * afterwards, to allow IndexWriter to free up disk space.</p>
+ * to the size of the index in your Directory: 2X if you are
+ * not using compound file format, and 3X if you are.
+ * For example, if your index size is 10 MB then you need
+ * an additional 20 MB free for this to complete (30 MB if
+ * you're using compound file format). This is also affected
+ * by the {@link Codec} that is used to execute the merge,
+ * and may result in even a bigger index. Also, it's best
+ * to call {@link #commit()} afterwards, to allow IndexWriter
+ * to free up disk space.</p>
*
* <p>If some but not all readers re-open while merging
* is underway, this will cause {@code > 2X} temporary
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Fri Apr 3 20:13:06 2015
@@ -232,8 +232,8 @@ public class PayloadNearQuery extends Sp
scratch.bytes = thePayload;
scratch.offset = 0;
scratch.length = thePayload.length;
- payloadScore = function.currentScore(doc, fieldName, start, end,
- payloadsSeen, payloadScore, docScorer.computePayloadFactor(doc,
+ payloadScore = function.currentScore(docID(), fieldName, start, end,
+ payloadsSeen, payloadScore, docScorer.computePayloadFactor(docID(),
spans.startPosition(), spans.endPosition(), scratch));
++payloadsSeen;
}
@@ -241,7 +241,7 @@ public class PayloadNearQuery extends Sp
//
@Override
- protected boolean setFreqCurrentDoc() throws IOException {
+ protected void setFreqCurrentDoc() throws IOException {
freq = 0.0f;
payloadScore = 0;
payloadsSeen = 0;
@@ -255,14 +255,12 @@ public class PayloadNearQuery extends Sp
getPayloads(spansArr);
startPos = spans.nextStartPosition();
} while (startPos != Spans.NO_MORE_POSITIONS);
- return true;
}
@Override
- public float score() throws IOException {
-
- return super.score()
- * function.docScore(doc, fieldName, payloadsSeen, payloadScore);
+ public float scoreCurrentDoc() throws IOException {
+ return super.scoreCurrentDoc()
+ * function.docScore(docID(), fieldName, payloadsSeen, payloadScore);
}
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Fri Apr 3 20:13:06 2015
@@ -99,7 +99,7 @@ public class PayloadTermQuery extends Sp
}
@Override
- protected boolean setFreqCurrentDoc() throws IOException {
+ protected void setFreqCurrentDoc() throws IOException {
freq = 0.0f;
numMatches = 0;
payloadScore = 0;
@@ -115,7 +115,6 @@ public class PayloadTermQuery extends Sp
startPos = spans.nextStartPosition();
} while (startPos != Spans.NO_MORE_POSITIONS);
- return freq != 0;
}
protected void processPayload(Similarity similarity) throws IOException {
@@ -123,11 +122,11 @@ public class PayloadTermQuery extends Sp
final PostingsEnum postings = termSpans.getPostings();
payload = postings.getPayload();
if (payload != null) {
- payloadScore = function.currentScore(doc, term.field(),
+ payloadScore = function.currentScore(docID(), term.field(),
spans.startPosition(), spans.endPosition(), payloadsSeen, payloadScore,
- docScorer.computePayloadFactor(doc, spans.startPosition(), spans.endPosition(), payload));
+ docScorer.computePayloadFactor(docID(), spans.startPosition(), spans.endPosition(), payload));
} else {
- payloadScore = function.currentScore(doc, term.field(),
+ payloadScore = function.currentScore(docID(), term.field(),
spans.startPosition(), spans.endPosition(), payloadsSeen, payloadScore, 1F);
}
payloadsSeen++;
@@ -143,8 +142,7 @@ public class PayloadTermQuery extends Sp
* @throws IOException if there is a low-level I/O error
*/
@Override
- public float score() throws IOException {
-
+ public float scoreCurrentDoc() throws IOException {
return includeSpanScore ? getSpanScore() * getPayloadScore()
: getPayloadScore();
}
@@ -160,7 +158,7 @@ public class PayloadTermQuery extends Sp
* @see #score()
*/
protected float getSpanScore() throws IOException {
- return super.score();
+ return super.scoreCurrentDoc();
}
/**
@@ -170,7 +168,7 @@ public class PayloadTermQuery extends Sp
* {@link PayloadFunction#docScore(int, String, int, float)}
*/
protected float getPayloadScore() {
- return function.docScore(doc, term.field(), payloadsSeen, payloadScore);
+ return function.docScore(docID(), term.field(), payloadsSeen, payloadScore);
}
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpans.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpans.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpans.java Fri Apr 3 20:13:06 2015
@@ -29,11 +29,11 @@ import java.util.Objects;
* Common super class for un/ordered Spans
*/
abstract class NearSpans extends Spans {
- SpanNearQuery query;
- int allowedSlop;
+ final SpanNearQuery query;
+ final int allowedSlop;
- List<Spans> subSpans; // in query order
- DocIdSetIterator conjunction; // use to move to next doc with all clauses
+ final Spans[] subSpans; // in query order
+ final DocIdSetIterator conjunction; // use to move to next doc with all clauses
boolean atFirstInCurrentDoc;
boolean oneExhaustedInCurrentDoc; // no more results possbile in current doc
@@ -44,7 +44,7 @@ abstract class NearSpans extends Spans {
if (subSpans.size() < 2) {
throw new IllegalArgumentException("Less than 2 subSpans: " + query);
}
- this.subSpans = Objects.requireNonNull(subSpans); // in query order
+ this.subSpans = subSpans.toArray(new Spans[subSpans.size()]); // in query order
this.conjunction = ConjunctionDISI.intersect(subSpans);
}
@@ -91,13 +91,8 @@ abstract class NearSpans extends Spans {
return res;
}
- private Spans[] subSpansArray = null; // init only when needed.
-
public Spans[] getSubSpans() {
- if (subSpansArray == null) {
- subSpansArray = subSpans.toArray(new Spans[subSpans.size()]);
- }
- return subSpansArray;
+ return subSpans;
}
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java Fri Apr 3 20:13:06 2015
@@ -18,12 +18,8 @@ package org.apache.lucene.search.spans;
*/
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.Collection;
-import java.util.Set;
/** A Spans that is formed from the ordered subspans of a SpanNearQuery
* where the subspans do not overlap and have a maximum slop between them,
@@ -146,11 +142,11 @@ public class NearSpansOrdered extends Ne
* otherwise at least one is exhausted in the current doc.
*/
private boolean stretchToOrder() throws IOException {
- Spans prevSpans = subSpans.get(0);
+ Spans prevSpans = subSpans[0];
assert prevSpans.startPosition() != NO_MORE_POSITIONS : "prevSpans no start position "+prevSpans;
assert prevSpans.endPosition() != NO_MORE_POSITIONS;
- for (int i = 1; i < subSpans.size(); i++) {
- Spans spans = subSpans.get(i);
+ for (int i = 1; i < subSpans.length; i++) {
+ Spans spans = subSpans[i];
assert spans.startPosition() != NO_MORE_POSITIONS;
assert spans.endPosition() != NO_MORE_POSITIONS;
@@ -169,15 +165,14 @@ public class NearSpansOrdered extends Ne
* on all subSpans, except the last one, in reverse order.
*/
protected boolean shrinkToAfterShortestMatch() throws IOException {
- Spans lastSubSpans = subSpans.get(subSpans.size() - 1);
+ Spans lastSubSpans = subSpans[subSpans.length - 1];
matchStart = lastSubSpans.startPosition();
matchEnd = lastSubSpans.endPosition();
int matchSlop = 0;
int lastStart = matchStart;
- int lastEnd = matchEnd;
- for (int i = subSpans.size() - 2; i >= 0; i--) {
- Spans prevSpans = subSpans.get(i);
+ for (int i = subSpans.length - 2; i >= 0; i--) {
+ Spans prevSpans = subSpans[i];
int prevStart = prevSpans.startPosition();
int prevEnd = prevSpans.endPosition();
@@ -206,7 +201,6 @@ public class NearSpansOrdered extends Ne
*/
matchStart = prevStart;
lastStart = prevStart;
- lastEnd = prevEnd;
}
boolean match = matchSlop <= allowedSlop;
@@ -224,16 +218,14 @@ public class NearSpansOrdered extends Ne
return atFirstInCurrentDoc ? -1 : matchEnd;
}
- /** Throws an UnsupportedOperationException */
@Override
public Collection<byte[]> getPayload() throws IOException {
- throw new UnsupportedOperationException("Use NearSpansPayloadOrdered instead");
+ return null;
}
- /** Throws an UnsupportedOperationException */
@Override
public boolean isPayloadAvailable() {
- throw new UnsupportedOperationException("Use NearSpansPayloadOrdered instead");
+ return false;
}
@Override
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansPayloadOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansPayloadOrdered.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansPayloadOrdered.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/NearSpansPayloadOrdered.java Fri Apr 3 20:13:06 2015
@@ -47,7 +47,7 @@ public class NearSpansPayloadOrdered ext
* Also collect the payloads.
*/
protected boolean shrinkToAfterShortestMatch() throws IOException {
- Spans lastSubSpans = subSpans.get(subSpans.size() - 1);
+ Spans lastSubSpans = subSpans[subSpans.length - 1];
matchStart = lastSubSpans.startPosition();
matchEnd = lastSubSpans.endPosition();
@@ -62,9 +62,8 @@ public class NearSpansPayloadOrdered ext
int matchSlop = 0;
int lastStart = matchStart;
- int lastEnd = matchEnd;
- for (int i = subSpans.size() - 2; i >= 0; i--) {
- Spans prevSpans = subSpans.get(i);
+ for (int i = subSpans.length - 2; i >= 0; i--) {
+ Spans prevSpans = subSpans[i];
if (prevSpans.isPayloadAvailable()) {
Collection<byte[]> payload = prevSpans.getPayload();
@@ -112,7 +111,6 @@ public class NearSpansPayloadOrdered ext
*/
matchStart = prevStart;
lastStart = prevStart;
- lastEnd = prevEnd;
}
boolean match = matchSlop <= allowedSlop;
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanNearQuery.java Fri Apr 3 20:13:06 2015
@@ -18,19 +18,17 @@ package org.apache.lucene.search.spans;
*/
import java.io.IOException;
-
-
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
-
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
@@ -131,10 +129,15 @@ public class SpanNearQuery extends SpanQ
return null; // all required
}
}
+
+ Terms terms = context.reader().terms(field);
+ if (terms == null) {
+ return null; // field does not exist
+ }
// all NearSpans require at least two subSpans
return (! inOrder) ? new NearSpansUnordered(this, subSpans)
- : collectPayloads ? new NearSpansPayloadOrdered(this, subSpans)
+ : collectPayloads && terms.hasPayloads() ? new NearSpansPayloadOrdered(this, subSpans)
: new NearSpansOrdered(this, subSpans);
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java Fri Apr 3 20:13:06 2015
@@ -146,7 +146,7 @@ public abstract class SpanPositionCheckQ
startPos = in.nextStartPosition();
assert startPos != NO_MORE_POSITIONS;
for (;;) {
- switch(acceptPosition(this)) {
+ switch(acceptPosition(in)) {
case YES:
atFirstInCurrentDoc = true;
return in.docID();
@@ -180,7 +180,7 @@ public abstract class SpanPositionCheckQ
if (startPos == NO_MORE_POSITIONS) {
return NO_MORE_POSITIONS;
}
- switch(acceptPosition(this)) {
+ switch(acceptPosition(in)) {
case YES:
return startPos;
case NO:
Modified: lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java Fri Apr 3 20:13:06 2015
@@ -21,48 +21,58 @@ import java.io.IOException;
import java.util.Objects;
import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.similarities.Similarity;
/**
* Public for extension only.
*/
public class SpanScorer extends Scorer {
- protected Spans spans;
+ /** underlying spans we are scoring from */
+ protected final Spans spans;
+ /** similarity used in default score impl */
+ protected final Similarity.SimScorer docScorer;
- protected int doc;
+ /** accumulated sloppy freq (computed in setFreqCurrentDoc) */
protected float freq;
+ /** number of matches (computed in setFreqCurrentDoc) */
protected int numMatches;
- protected final Similarity.SimScorer docScorer;
+
+ private int lastScoredDoc = -1; // last doc we called setFreqCurrentDoc() for
- protected SpanScorer(Spans spans, SpanWeight weight, Similarity.SimScorer docScorer)
- throws IOException {
+ protected SpanScorer(Spans spans, SpanWeight weight, Similarity.SimScorer docScorer) throws IOException {
super(weight);
this.docScorer = Objects.requireNonNull(docScorer);
this.spans = Objects.requireNonNull(spans);
- this.doc = -1;
}
@Override
- public int nextDoc() throws IOException {
- int prevDoc = doc;
- doc = spans.nextDoc();
- if (doc != NO_MORE_DOCS) {
- setFreqCurrentDoc();
- }
- return doc;
+ public final int nextDoc() throws IOException {
+ return spans.nextDoc();
}
@Override
- public int advance(int target) throws IOException {
- int prevDoc = doc;
- doc = spans.advance(target);
- if (doc != NO_MORE_DOCS) {
+ public final int advance(int target) throws IOException {
+ return spans.advance(target);
+ }
+
+ /**
+ * Ensure setFreqCurrentDoc is called, if not already called for the current doc.
+ */
+ private final void ensureFreq() throws IOException {
+ int currentDoc = spans.docID();
+ if (lastScoredDoc != currentDoc) {
setFreqCurrentDoc();
+ lastScoredDoc = currentDoc;
}
- return doc;
}
- protected boolean setFreqCurrentDoc() throws IOException {
+ /**
+ * Sets {@link #freq} and {@link #numMatches} for the current document.
+ * <p>
+ * This will be called at most once per document.
+ */
+ protected void setFreqCurrentDoc() throws IOException {
freq = 0.0f;
numMatches = 0;
@@ -90,34 +100,46 @@ public class SpanScorer extends Scorer {
assert spans.startPosition() == Spans.NO_MORE_POSITIONS : "incorrect final start position, spans="+spans;
assert spans.endPosition() == Spans.NO_MORE_POSITIONS : "incorrect final end position, spans="+spans;
-
- return true;
+ }
+
+ /**
+ * Score the current doc. The default implementation scores the doc
+ * with the similarity using the slop-adjusted {@link #freq}.
+ */
+ protected float scoreCurrentDoc() throws IOException {
+ return docScorer.score(spans.docID(), freq);
}
@Override
- public int docID() { return doc; }
+ public final int docID() { return spans.docID(); }
@Override
- public float score() throws IOException {
- float s = docScorer.score(doc, freq);
- return s;
+ public final float score() throws IOException {
+ ensureFreq();
+ return scoreCurrentDoc();
}
@Override
- public int freq() throws IOException {
+ public final int freq() throws IOException {
+ ensureFreq();
return numMatches;
}
/** Returns the intermediate "sloppy freq" adjusted for edit distance
* @lucene.internal */
// only public so .payloads can see it.
- public float sloppyFreq() throws IOException {
+ public final float sloppyFreq() throws IOException {
+ ensureFreq();
return freq;
}
@Override
- public long cost() {
+ public final long cost() {
return spans.cost();
}
+ @Override
+ public final TwoPhaseIterator asTwoPhaseIterator() {
+ return spans.asTwoPhaseIterator();
+ }
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterForceMerge.java Fri Apr 3 20:13:06 2015
@@ -199,8 +199,8 @@ public class TestIndexWriterForceMerge e
assertTrue("forceMerge used too much temporary space: starting usage was "
+ startDiskUsage + " bytes; final usage was " + finalDiskUsage
+ " bytes; max temp usage was " + maxDiskUsage
- + " but should have been " + (3 * maxStartFinalDiskUsage)
- + " (= 3X starting usage), BEFORE=" + startListing + "AFTER=" + listFiles(dir), maxDiskUsage <= 3 * maxStartFinalDiskUsage);
+ + " but should have been at most " + (4 * maxStartFinalDiskUsage)
+ + " (= 4X starting usage), BEFORE=" + startListing + "AFTER=" + listFiles(dir), maxDiskUsage <= 4 * maxStartFinalDiskUsage);
dir.close();
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java Fri Apr 3 20:13:06 2015
@@ -162,7 +162,12 @@ final class JustCompileSearchSpans {
}
@Override
- protected boolean setFreqCurrentDoc() {
+ protected void setFreqCurrentDoc() {
+ throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+ }
+
+ @Override
+ protected float scoreCurrentDoc() throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanSearchEquivalence.java Fri Apr 3 20:13:06 2015
@@ -21,6 +21,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.SearchEquivalenceTestBase;
import org.apache.lucene.search.TermQuery;
@@ -106,4 +107,122 @@ public class TestSpanSearchEquivalence e
SpanNearQuery q2 = new SpanNearQuery(subquery, 3, false);
assertSubsetOf(q1, q2);
}
+
+ /** SpanNearQuery([A B], N, false) â SpanNearQuery([A B], N+1, false) */
+ public void testSpanNearIncreasingSloppiness() throws Exception {
+ Term t1 = randomTerm();
+ Term t2 = randomTerm();
+ SpanQuery subquery[] = new SpanQuery[] { new SpanTermQuery(t1), new SpanTermQuery(t2) };
+ for (int i = 0; i < 10; i++) {
+ SpanNearQuery q1 = new SpanNearQuery(subquery, i, false);
+ SpanNearQuery q2 = new SpanNearQuery(subquery, i+1, false);
+ assertSubsetOf(q1, q2);
+ }
+ }
+
+ /** SpanNearQuery([A B C], N, false) â SpanNearQuery([A B C], N+1, false) */
+ public void testSpanNearIncreasingSloppiness3() throws Exception {
+ Term t1 = randomTerm();
+ Term t2 = randomTerm();
+ Term t3 = randomTerm();
+ SpanQuery subquery[] = new SpanQuery[] { new SpanTermQuery(t1), new SpanTermQuery(t2), new SpanTermQuery(t3) };
+ for (int i = 0; i < 10; i++) {
+ SpanNearQuery q1 = new SpanNearQuery(subquery, i, false);
+ SpanNearQuery q2 = new SpanNearQuery(subquery, i+1, false);
+ assertSubsetOf(q1, q2);
+ }
+ }
+
+ /** SpanNearQuery([A B], N, true) â SpanNearQuery([A B], N+1, true) */
+ public void testSpanNearIncreasingOrderedSloppiness() throws Exception {
+ Term t1 = randomTerm();
+ Term t2 = randomTerm();
+ SpanQuery subquery[] = new SpanQuery[] { new SpanTermQuery(t1), new SpanTermQuery(t2) };
+ for (int i = 0; i < 10; i++) {
+ SpanNearQuery q1 = new SpanNearQuery(subquery, i, false);
+ SpanNearQuery q2 = new SpanNearQuery(subquery, i+1, false);
+ assertSubsetOf(q1, q2);
+ }
+ }
+
+ /** SpanNearQuery([A B C], N, true) â SpanNearQuery([A B C], N+1, true) */
+ public void testSpanNearIncreasingOrderedSloppiness3() throws Exception {
+ Term t1 = randomTerm();
+ Term t2 = randomTerm();
+ Term t3 = randomTerm();
+ SpanQuery subquery[] = new SpanQuery[] { new SpanTermQuery(t1), new SpanTermQuery(t2), new SpanTermQuery(t3) };
+ for (int i = 0; i < 10; i++) {
+ SpanNearQuery q1 = new SpanNearQuery(subquery, i, true);
+ SpanNearQuery q2 = new SpanNearQuery(subquery, i+1, true);
+ assertSubsetOf(q1, q2);
+ }
+ }
+
+ /** SpanFirstQuery(A, N) â TermQuery(A) */
+ public void testSpanFirstTerm() throws Exception {
+ Term t1 = randomTerm();
+ for (int i = 0; i < 10; i++) {
+ Query q1 = new SpanFirstQuery(new SpanTermQuery(t1), i);
+ Query q2 = new TermQuery(t1);
+ assertSubsetOf(q1, q2);
+ }
+ }
+
+ /** SpanFirstQuery(A, N) â SpanFirstQuery(A, N+1) */
+ public void testSpanFirstTermIncreasing() throws Exception {
+ Term t1 = randomTerm();
+ for (int i = 0; i < 10; i++) {
+ Query q1 = new SpanFirstQuery(new SpanTermQuery(t1), i);
+ Query q2 = new SpanFirstQuery(new SpanTermQuery(t1), i+1);
+ assertSubsetOf(q1, q2);
+ }
+ }
+
+ /** SpanFirstQuery(A, â) = TermQuery(A) */
+ public void testSpanFirstTermEverything() throws Exception {
+ Term t1 = randomTerm();
+ Query q1 = new SpanFirstQuery(new SpanTermQuery(t1), Integer.MAX_VALUE);
+ Query q2 = new TermQuery(t1);
+ assertSameSet(q1, q2);
+ }
+
+ /** SpanFirstQuery([A B], N) â SpanNearQuery([A B]) */
+ @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6393")
+ public void testSpanFirstNear() throws Exception {
+ Term t1 = randomTerm();
+ Term t2 = randomTerm();
+ SpanQuery subquery[] = new SpanQuery[] { new SpanTermQuery(t1), new SpanTermQuery(t2) };
+ SpanQuery nearQuery = new SpanNearQuery(subquery, 10, true);
+ for (int i = 0; i < 10; i++) {
+ Query q1 = new SpanFirstQuery(nearQuery, i);
+ Query q2 = nearQuery;
+ assertSubsetOf(q1, q2);
+ }
+ }
+
+ /** SpanFirstQuery([A B], N) â SpanFirstQuery([A B], N+1) */
+ @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6393")
+ public void testSpanFirstNearIncreasing() throws Exception {
+ Term t1 = randomTerm();
+ Term t2 = randomTerm();
+ SpanQuery subquery[] = new SpanQuery[] { new SpanTermQuery(t1), new SpanTermQuery(t2) };
+ SpanQuery nearQuery = new SpanNearQuery(subquery, 10, true);
+ for (int i = 0; i < 10; i++) {
+ Query q1 = new SpanFirstQuery(nearQuery, i);
+ Query q2 = new SpanFirstQuery(nearQuery, i+1);
+ assertSubsetOf(q1, q2);
+ }
+ }
+
+ /** SpanFirstQuery([A B], â) = SpanNearQuery([A B]) */
+ @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-6393")
+ public void testSpanFirstNearEverything() throws Exception {
+ Term t1 = randomTerm();
+ Term t2 = randomTerm();
+ SpanQuery subquery[] = new SpanQuery[] { new SpanTermQuery(t1), new SpanTermQuery(t2) };
+ SpanQuery nearQuery = new SpanNearQuery(subquery, 10, true);
+ Query q1 = new SpanFirstQuery(nearQuery, Integer.MAX_VALUE);
+ Query q2 = nearQuery;
+ assertSameSet(q1, q2);
+ }
}
Modified: lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Fri Apr 3 20:13:06 2015
@@ -306,6 +306,9 @@ public class WeightedSpanTermExtractor {
}
Bits acceptDocs = context.reader().getLiveDocs();
final Spans spans = q.getSpans(context, acceptDocs, termContexts);
+ if (spans == null) {
+ return;
+ }
// collect span positions
while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
Modified: lucene/dev/branches/lucene6271/lucene/ivy-versions.properties
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/ivy-versions.properties?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/ivy-versions.properties (original)
+++ lucene/dev/branches/lucene6271/lucene/ivy-versions.properties Fri Apr 3 20:13:06 2015
@@ -116,9 +116,9 @@ org.apache.hadoop.version = 2.6.0
# The httpcore version is often different from the httpclient and httpmime versions,
# so the httpcore version value should not share the same symbolic name with them.
-/org.apache.httpcomponents/httpclient = 4.3.1
-/org.apache.httpcomponents/httpcore = 4.3
-/org.apache.httpcomponents/httpmime = 4.3.1
+/org.apache.httpcomponents/httpclient = 4.4.1
+/org.apache.httpcomponents/httpcore = 4.4.1
+/org.apache.httpcomponents/httpmime = 4.4.1
/org.apache.ivy/ivy = 2.3.0
Modified: lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java (original)
+++ lucene/dev/branches/lucene6271/lucene/join/src/java/org/apache/lucene/search/join/JoinUtil.java Fri Apr 3 20:13:06 2015
@@ -17,7 +17,12 @@ package org.apache.lucene.search.join;
* limitations under the License.
*/
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import java.io.IOException;
@@ -90,4 +95,78 @@ public final class JoinUtil {
}
}
+ /**
+ * A query time join using global ordinals over a dedicated join field.
+ *
+ * This join has certain restrictions and requirements:
+ * 1) A document can only refer to one other document. (but can be referred by one or more documents)
+ * 2) Documents on each side of the join must be distinguishable. Typically this can be done by adding an extra field
+ * that identifies the "from" and "to" side and then the fromQuery and toQuery must take the this into account.
+ * 3) There must be a single sorted doc values join field used by both the "from" and "to" documents. This join field
+ * should store the join values as UTF-8 strings.
+ * 4) An ordinal map must be provided that is created on top of the join field.
+ *
+ * @param joinField The {@link org.apache.lucene.index.SortedDocValues} field containing the join values
+ * @param fromQuery The query containing the actual user query. Also the fromQuery can only match "from" documents.
+ * @param toQuery The query identifying all documents on the "to" side.
+ * @param searcher The index searcher used to execute the from query
+ * @param scoreMode Instructs how scores from the fromQuery are mapped to the returned query
+ * @param ordinalMap The ordinal map constructed over the joinField. In case of a single segment index, no ordinal map
+ * needs to be provided.
+ * @return a {@link Query} instance that can be used to join documents based on the join field
+ * @throws IOException If I/O related errors occur
+ */
+ public static Query createJoinQuery(String joinField,
+ Query fromQuery,
+ Query toQuery,
+ IndexSearcher searcher,
+ ScoreMode scoreMode,
+ MultiDocValues.OrdinalMap ordinalMap) throws IOException {
+ IndexReader indexReader = searcher.getIndexReader();
+ int numSegments = indexReader.leaves().size();
+ final long valueCount;
+ if (numSegments == 0) {
+ return new MatchNoDocsQuery();
+ } else if (numSegments == 1) {
+ // No need to use the ordinal map, because there is just one segment.
+ ordinalMap = null;
+ LeafReader leafReader = searcher.getIndexReader().leaves().get(0).reader();
+ SortedDocValues joinSortedDocValues = leafReader.getSortedDocValues(joinField);
+ if (joinSortedDocValues != null) {
+ valueCount = joinSortedDocValues.getValueCount();
+ } else {
+ return new MatchNoDocsQuery();
+ }
+ } else {
+ if (ordinalMap == null) {
+ throw new IllegalArgumentException("OrdinalMap is required, because there is more than 1 segment");
+ }
+ valueCount = ordinalMap.getValueCount();
+ }
+
+ Query rewrittenFromQuery = searcher.rewrite(fromQuery);
+ if (scoreMode == ScoreMode.None) {
+ GlobalOrdinalsCollector globalOrdinalsCollector = new GlobalOrdinalsCollector(joinField, ordinalMap, valueCount);
+ searcher.search(fromQuery, globalOrdinalsCollector);
+ return new GlobalOrdinalsQuery(globalOrdinalsCollector.getCollectorOrdinals(), joinField, ordinalMap, toQuery, rewrittenFromQuery, indexReader);
+ }
+
+ GlobalOrdinalsWithScoreCollector globalOrdinalsWithScoreCollector;
+ switch (scoreMode) {
+ case Total:
+ globalOrdinalsWithScoreCollector = new GlobalOrdinalsWithScoreCollector.Sum(joinField, ordinalMap, valueCount);
+ break;
+ case Max:
+ globalOrdinalsWithScoreCollector = new GlobalOrdinalsWithScoreCollector.Max(joinField, ordinalMap, valueCount);
+ break;
+ case Avg:
+ globalOrdinalsWithScoreCollector = new GlobalOrdinalsWithScoreCollector.Avg(joinField, ordinalMap, valueCount);
+ break;
+ default:
+ throw new IllegalArgumentException(String.format(Locale.ROOT, "Score mode %s isn't supported.", scoreMode));
+ }
+ searcher.search(fromQuery, globalOrdinalsWithScoreCollector);
+ return new GlobalOrdinalsWithScoreQuery(globalOrdinalsWithScoreCollector, joinField, ordinalMap, toQuery, rewrittenFromQuery, indexReader);
+ }
+
}
Modified: lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java (original)
+++ lucene/dev/branches/lucene6271/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java Fri Apr 3 20:13:06 2015
@@ -17,19 +17,6 @@ package org.apache.lucene.search.join;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
@@ -38,27 +25,29 @@ import org.apache.lucene.document.Sorted
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.FilterLeafCollector;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Scorer;
@@ -74,8 +63,22 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
+import org.apache.lucene.util.packed.PackedInts;
import org.junit.Test;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
public class TestJoinUtil extends LuceneTestCase {
public void testSimple() throws Exception {
@@ -169,6 +172,180 @@ public class TestJoinUtil extends Lucene
dir.close();
}
+ public void testSimpleOrdinalsJoin() throws Exception {
+ final String idField = "id";
+ final String productIdField = "productId";
+ // A field indicating to what type a document belongs, which is then used to distinques between documents during joining.
+ final String typeField = "type";
+ // A single sorted doc values field that holds the join values for all document types.
+ // Typically during indexing a schema will automatically create this field with the values
+ final String joinField = idField + productIdField;
+
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(
+ random(),
+ dir,
+ newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE));
+
+ // 0
+ Document doc = new Document();
+ doc.add(new TextField(idField, "1", Field.Store.NO));
+ doc.add(new TextField(typeField, "product", Field.Store.NO));
+ doc.add(new TextField("description", "random text", Field.Store.NO));
+ doc.add(new TextField("name", "name1", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ // 1
+ doc = new Document();
+ doc.add(new TextField(productIdField, "1", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "10.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ // 2
+ doc = new Document();
+ doc.add(new TextField(productIdField, "1", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "20.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("1")));
+ w.addDocument(doc);
+
+ // 3
+ doc = new Document();
+ doc.add(new TextField(idField, "2", Field.Store.NO));
+ doc.add(new TextField(typeField, "product", Field.Store.NO));
+ doc.add(new TextField("description", "more random text", Field.Store.NO));
+ doc.add(new TextField("name", "name2", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));
+ w.addDocument(doc);
+ w.commit();
+
+ // 4
+ doc = new Document();
+ doc.add(new TextField(productIdField, "2", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "10.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));
+ w.addDocument(doc);
+
+ // 5
+ doc = new Document();
+ doc.add(new TextField(productIdField, "2", Field.Store.NO));
+ doc.add(new TextField(typeField, "price", Field.Store.NO));
+ doc.add(new TextField("price", "20.0", Field.Store.NO));
+ doc.add(new SortedDocValuesField(joinField, new BytesRef("2")));
+ w.addDocument(doc);
+
+ IndexSearcher indexSearcher = new IndexSearcher(w.getReader());
+ w.close();
+
+ IndexReader r = indexSearcher.getIndexReader();
+ SortedDocValues[] values = new SortedDocValues[r.leaves().size()];
+ for (int i = 0; i < values.length; i++) {
+ LeafReader leafReader = r.leaves().get(i).reader();
+ values[i] = DocValues.getSorted(leafReader, joinField);
+ }
+ MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(
+ r.getCoreCacheKey(), values, PackedInts.DEFAULT
+ );
+
+ Query toQuery = new TermQuery(new Term(typeField, "price"));
+ Query fromQuery = new TermQuery(new Term("name", "name2"));
+ // Search for product and return prices
+ Query joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, ScoreMode.None, ordinalMap);
+ TopDocs result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(4, result.scoreDocs[0].doc);
+ assertEquals(5, result.scoreDocs[1].doc);
+
+ fromQuery = new TermQuery(new Term("name", "name1"));
+ joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, ScoreMode.None, ordinalMap);
+ result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(1, result.scoreDocs[0].doc);
+ assertEquals(2, result.scoreDocs[1].doc);
+
+ // Search for prices and return products
+ fromQuery = new TermQuery(new Term("price", "20.0"));
+ toQuery = new TermQuery(new Term(typeField, "product"));
+ joinQuery = JoinUtil.createJoinQuery(joinField, fromQuery, toQuery, indexSearcher, ScoreMode.None, ordinalMap);
+ result = indexSearcher.search(joinQuery, 10);
+ assertEquals(2, result.totalHits);
+ assertEquals(0, result.scoreDocs[0].doc);
+ assertEquals(3, result.scoreDocs[1].doc);
+
+ indexSearcher.getIndexReader().close();
+ dir.close();
+ }
+
+ public void testRandomOrdinalsJoin() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(
+ random(),
+ dir,
+ newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy())
+ );
+ IndexIterationContext context = createContext(100, w, false, true);
+
+ w.forceMerge(1);
+
+ w.close();
+ IndexReader topLevelReader = DirectoryReader.open(dir);
+
+ SortedDocValues[] values = new SortedDocValues[topLevelReader.leaves().size()];
+ for (LeafReaderContext leadContext : topLevelReader.leaves()) {
+ values[leadContext.ord] = DocValues.getSorted(leadContext.reader(), "join_field");
+ }
+ context.ordinalMap = MultiDocValues.OrdinalMap.build(
+ topLevelReader.getCoreCacheKey(), values, PackedInts.DEFAULT
+ );
+ IndexSearcher indexSearcher = newSearcher(topLevelReader);
+
+ int r = random().nextInt(context.randomUniqueValues.length);
+ boolean from = context.randomFrom[r];
+ String randomValue = context.randomUniqueValues[r];
+ BitSet expectedResult = createExpectedResult(randomValue, from, indexSearcher.getIndexReader(), context);
+
+ final Query actualQuery = new TermQuery(new Term("value", randomValue));
+ if (VERBOSE) {
+ System.out.println("actualQuery=" + actualQuery);
+ }
+ final ScoreMode scoreMode = ScoreMode.values()[random().nextInt(ScoreMode.values().length)];
+ if (VERBOSE) {
+ System.out.println("scoreMode=" + scoreMode);
+ }
+
+ final Query joinQuery;
+ if (from) {
+ BooleanQuery fromQuery = new BooleanQuery();
+ fromQuery.add(new TermQuery(new Term("type", "from")), BooleanClause.Occur.FILTER);
+ fromQuery.add(actualQuery, BooleanClause.Occur.MUST);
+ Query toQuery = new TermQuery(new Term("type", "to"));
+ joinQuery = JoinUtil.createJoinQuery("join_field", fromQuery, toQuery, indexSearcher, scoreMode, context.ordinalMap);
+ } else {
+ BooleanQuery fromQuery = new BooleanQuery();
+ fromQuery.add(new TermQuery(new Term("type", "to")), BooleanClause.Occur.FILTER);
+ fromQuery.add(actualQuery, BooleanClause.Occur.MUST);
+ Query toQuery = new TermQuery(new Term("type", "from"));
+ joinQuery = JoinUtil.createJoinQuery("join_field", fromQuery, toQuery, indexSearcher, scoreMode, context.ordinalMap);
+ }
+ if (VERBOSE) {
+ System.out.println("joinQuery=" + joinQuery);
+ }
+
+ final BitSet actualResult = new FixedBitSet(indexSearcher.getIndexReader().maxDoc());
+ final TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.create(10);
+ indexSearcher.search(joinQuery, MultiCollector.wrap(new BitSetCollector(actualResult), topScoreDocCollector));
+ assertBitSet(expectedResult, actualResult, indexSearcher);
+ TopDocs expectedTopDocs = createExpectedTopDocs(randomValue, from, scoreMode, context);
+ TopDocs actualTopDocs = topScoreDocCollector.topDocs();
+ assertTopDocs(expectedTopDocs, actualTopDocs, scoreMode, indexSearcher, joinQuery);
+ topLevelReader.close();
+ dir.close();
+ }
+
// TermsWithScoreCollector.MV.Avg forgets to grow beyond TermsWithScoreCollector.INITIAL_ARRAY_SIZE
public void testOverflowTermsWithScoreCollector() throws Exception {
test300spartans(true, ScoreMode.Avg);
@@ -218,7 +395,7 @@ public class TestJoinUtil extends Lucene
TopDocs result = indexSearcher.search(joinQuery, 10);
assertEquals(1, result.totalHits);
assertEquals(0, result.scoreDocs[0].doc);
-
+
indexSearcher.getIndexReader().close();
dir.close();
@@ -310,7 +487,7 @@ public class TestJoinUtil extends Lucene
assertFalse("optimized bulkScorer was not used for join query embedded in boolean query!", sawFive);
}
}
-
+
@Override
public boolean needsScores() {
return false;
@@ -448,7 +625,7 @@ public class TestJoinUtil extends Lucene
dir,
newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy())
);
- IndexIterationContext context = createContext(numberOfDocumentsToIndex, w, multipleValuesPerDocument);
+ IndexIterationContext context = createContext(numberOfDocumentsToIndex, w, multipleValuesPerDocument, false);
IndexReader topLevelReader = w.getReader();
w.close();
@@ -485,73 +662,64 @@ public class TestJoinUtil extends Lucene
// Need to know all documents that have matches. TopDocs doesn't give me that and then I'd be also testing TopDocsCollector...
final BitSet actualResult = new FixedBitSet(indexSearcher.getIndexReader().maxDoc());
final TopScoreDocCollector topScoreDocCollector = TopScoreDocCollector.create(10);
- indexSearcher.search(joinQuery, new Collector() {
-
- @Override
- public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
- final int docBase = context.docBase;
- final LeafCollector in = topScoreDocCollector.getLeafCollector(context);
- return new FilterLeafCollector(in) {
-
- @Override
- public void collect(int doc) throws IOException {
- super.collect(doc);
- actualResult.set(doc + docBase);
- }
- };
- }
-
- @Override
- public boolean needsScores() {
- return topScoreDocCollector.needsScores();
- }
- });
+ indexSearcher.search(joinQuery, MultiCollector.wrap(new BitSetCollector(actualResult), topScoreDocCollector));
// Asserting bit set...
- if (VERBOSE) {
- System.out.println("expected cardinality:" + expectedResult.cardinality());
- DocIdSetIterator iterator = new BitSetIterator(expectedResult, expectedResult.cardinality());
- for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
- System.out.println(String.format(Locale.ROOT, "Expected doc[%d] with id value %s", doc, indexSearcher.doc(doc).get("id")));
- }
- System.out.println("actual cardinality:" + actualResult.cardinality());
- iterator = new BitSetIterator(actualResult, actualResult.cardinality());
- for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
- System.out.println(String.format(Locale.ROOT, "Actual doc[%d] with id value %s", doc, indexSearcher.doc(doc).get("id")));
- }
- }
- assertEquals(expectedResult, actualResult);
-
+ assertBitSet(expectedResult, actualResult, indexSearcher);
// Asserting TopDocs...
TopDocs expectedTopDocs = createExpectedTopDocs(randomValue, from, scoreMode, context);
TopDocs actualTopDocs = topScoreDocCollector.topDocs();
- assertEquals(expectedTopDocs.totalHits, actualTopDocs.totalHits);
- assertEquals(expectedTopDocs.scoreDocs.length, actualTopDocs.scoreDocs.length);
- if (scoreMode == ScoreMode.None) {
- continue;
- }
-
- assertEquals(expectedTopDocs.getMaxScore(), actualTopDocs.getMaxScore(), 0.0f);
- for (int i = 0; i < expectedTopDocs.scoreDocs.length; i++) {
- if (VERBOSE) {
- System.out.printf(Locale.ENGLISH, "Expected doc: %d | Actual doc: %d\n", expectedTopDocs.scoreDocs[i].doc, actualTopDocs.scoreDocs[i].doc);
- System.out.printf(Locale.ENGLISH, "Expected score: %f | Actual score: %f\n", expectedTopDocs.scoreDocs[i].score, actualTopDocs.scoreDocs[i].score);
- }
- assertEquals(expectedTopDocs.scoreDocs[i].doc, actualTopDocs.scoreDocs[i].doc);
- assertEquals(expectedTopDocs.scoreDocs[i].score, actualTopDocs.scoreDocs[i].score, 0.0f);
- Explanation explanation = indexSearcher.explain(joinQuery, expectedTopDocs.scoreDocs[i].doc);
- assertEquals(expectedTopDocs.scoreDocs[i].score, explanation.getValue(), 0.0f);
- }
+ assertTopDocs(expectedTopDocs, actualTopDocs, scoreMode, indexSearcher, joinQuery);
}
topLevelReader.close();
dir.close();
}
}
- private IndexIterationContext createContext(int nDocs, RandomIndexWriter writer, boolean multipleValuesPerDocument) throws IOException {
- return createContext(nDocs, writer, writer, multipleValuesPerDocument);
+ private void assertBitSet(BitSet expectedResult, BitSet actualResult, IndexSearcher indexSearcher) throws IOException {
+ if (VERBOSE) {
+ System.out.println("expected cardinality:" + expectedResult.cardinality());
+ DocIdSetIterator iterator = new BitSetIterator(expectedResult, expectedResult.cardinality());
+ for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
+ System.out.println(String.format(Locale.ROOT, "Expected doc[%d] with id value %s", doc, indexSearcher.doc(doc).get("id")));
+ }
+ System.out.println("actual cardinality:" + actualResult.cardinality());
+ iterator = new BitSetIterator(actualResult, actualResult.cardinality());
+ for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
+ System.out.println(String.format(Locale.ROOT, "Actual doc[%d] with id value %s", doc, indexSearcher.doc(doc).get("id")));
+ }
+ }
+ assertEquals(expectedResult, actualResult);
+ }
+
+ private void assertTopDocs(TopDocs expectedTopDocs, TopDocs actualTopDocs, ScoreMode scoreMode, IndexSearcher indexSearcher, Query joinQuery) throws IOException {
+ assertEquals(expectedTopDocs.totalHits, actualTopDocs.totalHits);
+ assertEquals(expectedTopDocs.scoreDocs.length, actualTopDocs.scoreDocs.length);
+ if (scoreMode == ScoreMode.None) {
+ return;
+ }
+
+ assertEquals(expectedTopDocs.getMaxScore(), actualTopDocs.getMaxScore(), 0.0f);
+ for (int i = 0; i < expectedTopDocs.scoreDocs.length; i++) {
+ if (VERBOSE) {
+ System.out.printf(Locale.ENGLISH, "Expected doc: %d | Actual doc: %d\n", expectedTopDocs.scoreDocs[i].doc, actualTopDocs.scoreDocs[i].doc);
+ System.out.printf(Locale.ENGLISH, "Expected score: %f | Actual score: %f\n", expectedTopDocs.scoreDocs[i].score, actualTopDocs.scoreDocs[i].score);
+ }
+ assertEquals(expectedTopDocs.scoreDocs[i].doc, actualTopDocs.scoreDocs[i].doc);
+ assertEquals(expectedTopDocs.scoreDocs[i].score, actualTopDocs.scoreDocs[i].score, 0.0f);
+ Explanation explanation = indexSearcher.explain(joinQuery, expectedTopDocs.scoreDocs[i].doc);
+ assertEquals(expectedTopDocs.scoreDocs[i].score, explanation.getValue(), 0.0f);
+ }
+ }
+
+ private IndexIterationContext createContext(int nDocs, RandomIndexWriter writer, boolean multipleValuesPerDocument, boolean ordinalJoin) throws IOException {
+ return createContext(nDocs, writer, writer, multipleValuesPerDocument, ordinalJoin);
}
- private IndexIterationContext createContext(int nDocs, RandomIndexWriter fromWriter, RandomIndexWriter toWriter, boolean multipleValuesPerDocument) throws IOException {
+ private IndexIterationContext createContext(int nDocs, RandomIndexWriter fromWriter, RandomIndexWriter toWriter, boolean multipleValuesPerDocument, boolean globalOrdinalJoin) throws IOException {
+ if (globalOrdinalJoin) {
+ assertFalse("ordinal join doesn't support multiple join values per document", multipleValuesPerDocument);
+ }
+
IndexIterationContext context = new IndexIterationContext();
int numRandomValues = nDocs / 2;
context.randomUniqueValues = new String[numRandomValues];
@@ -560,8 +728,8 @@ public class TestJoinUtil extends Lucene
for (int i = 0; i < numRandomValues; i++) {
String uniqueRandomValue;
do {
- uniqueRandomValue = TestUtil.randomRealisticUnicodeString(random());
-// uniqueRandomValue = _TestUtil.randomSimpleString(random);
+// uniqueRandomValue = TestUtil.randomRealisticUnicodeString(random());
+ uniqueRandomValue = TestUtil.randomSimpleString(random());
} while ("".equals(uniqueRandomValue) || trackSet.contains(uniqueRandomValue));
// Generate unique values and empty strings aren't allowed.
trackSet.add(uniqueRandomValue);
@@ -581,15 +749,18 @@ public class TestJoinUtil extends Lucene
boolean from = context.randomFrom[randomI];
int numberOfLinkValues = multipleValuesPerDocument ? 2 + random().nextInt(10) : 1;
docs[i] = new RandomDoc(id, numberOfLinkValues, value, from);
+ if (globalOrdinalJoin) {
+ document.add(newStringField("type", from ? "from" : "to", Field.Store.NO));
+ }
for (int j = 0; j < numberOfLinkValues; j++) {
String linkValue = context.randomUniqueValues[random().nextInt(context.randomUniqueValues.length)];
docs[i].linkValues.add(linkValue);
if (from) {
if (!context.fromDocuments.containsKey(linkValue)) {
- context.fromDocuments.put(linkValue, new ArrayList<RandomDoc>());
+ context.fromDocuments.put(linkValue, new ArrayList<>());
}
if (!context.randomValueFromDocs.containsKey(value)) {
- context.randomValueFromDocs.put(value, new ArrayList<RandomDoc>());
+ context.randomValueFromDocs.put(value, new ArrayList<>());
}
context.fromDocuments.get(linkValue).add(docs[i]);
@@ -600,12 +771,15 @@ public class TestJoinUtil extends Lucene
} else {
document.add(new SortedDocValuesField("from", new BytesRef(linkValue)));
}
+ if (globalOrdinalJoin) {
+ document.add(new SortedDocValuesField("join_field", new BytesRef(linkValue)));
+ }
} else {
if (!context.toDocuments.containsKey(linkValue)) {
- context.toDocuments.put(linkValue, new ArrayList<RandomDoc>());
+ context.toDocuments.put(linkValue, new ArrayList<>());
}
if (!context.randomValueToDocs.containsKey(value)) {
- context.randomValueToDocs.put(value, new ArrayList<RandomDoc>());
+ context.randomValueToDocs.put(value, new ArrayList<>());
}
context.toDocuments.get(linkValue).add(docs[i]);
@@ -616,6 +790,9 @@ public class TestJoinUtil extends Lucene
} else {
document.add(new SortedDocValuesField("to", new BytesRef(linkValue)));
}
+ if (globalOrdinalJoin) {
+ document.add(new SortedDocValuesField("join_field", new BytesRef(linkValue)));
+ }
}
}
@@ -707,6 +884,9 @@ public class TestJoinUtil extends Lucene
if (joinScore == null) {
joinValueToJoinScores.put(BytesRef.deepCopyOf(joinValue), joinScore = new JoinScore());
}
+ if (VERBOSE) {
+ System.out.println("expected val=" + joinValue.utf8ToString() + " expected score=" + scorer.score());
+ }
joinScore.addScore(scorer.score());
}
@@ -720,7 +900,7 @@ public class TestJoinUtil extends Lucene
public void setScorer(Scorer scorer) {
this.scorer = scorer;
}
-
+
@Override
public boolean needsScores() {
return true;
@@ -777,7 +957,7 @@ public class TestJoinUtil extends Lucene
@Override
public void setScorer(Scorer scorer) {}
-
+
@Override
public boolean needsScores() {
return false;
@@ -875,6 +1055,7 @@ public class TestJoinUtil extends Lucene
Map<String, Map<Integer, JoinScore>> fromHitsToJoinScore = new HashMap<>();
Map<String, Map<Integer, JoinScore>> toHitsToJoinScore = new HashMap<>();
+ MultiDocValues.OrdinalMap ordinalMap;
}
private static class RandomDoc {
@@ -922,4 +1103,29 @@ public class TestJoinUtil extends Lucene
}
+ private static class BitSetCollector extends SimpleCollector {
+
+ private final BitSet bitSet;
+ private int docBase;
+
+ private BitSetCollector(BitSet bitSet) {
+ this.bitSet = bitSet;
+ }
+
+ @Override
+ public void collect(int doc) throws IOException {
+ bitSet.set(docBase + doc);
+ }
+
+ @Override
+ protected void doSetNextReader(LeafReaderContext context) throws IOException {
+ docBase = context.docBase;
+ }
+
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+ }
+
}
Modified: lucene/dev/branches/lucene6271/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/SuggestFieldTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/SuggestFieldTest.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/SuggestFieldTest.java (original)
+++ lucene/dev/branches/lucene6271/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/SuggestFieldTest.java Fri Apr 3 20:13:06 2015
@@ -43,7 +43,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.TermsQuery;
@@ -57,11 +56,9 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.TestUtil;
import org.junit.After;
import org.junit.Before;
@@ -158,9 +155,10 @@ public class SuggestFieldTest extends Lu
weights[i] = Math.abs(random().nextLong());
document.add(newSuggestField("suggest_field", "abc", weights[i]));
iw.addDocument(document);
- }
- if (rarely()) {
- iw.commit();
+
+ if (usually()) {
+ iw.commit();
+ }
}
DirectoryReader reader = iw.getReader();
@@ -200,11 +198,15 @@ public class SuggestFieldTest extends Lu
}
iw.addDocument(document);
document.clear();
+
+ if (usually()) {
+ iw.commit();
+ }
}
iw.deleteDocuments(new Term("str_field", "delete"));
- DirectoryReader reader = DirectoryReader.open(iw, false);
+ DirectoryReader reader = DirectoryReader.open(iw, true);
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader, analyzer);
TopSuggestDocs suggest = indexSearcher.suggest("suggest_field", "abc_", numLive);
assertSuggestions(suggest, expectedEntries.toArray(new Entry[expectedEntries.size()]));
@@ -224,6 +226,10 @@ public class SuggestFieldTest extends Lu
document.add(newStringField("str_fld", "deleted", Field.Store.NO));
iw.addDocument(document);
document.clear();
+
+ if (usually()) {
+ iw.commit();
+ }
}
Filter filter = new QueryWrapperFilter(new TermsQuery("str_fld", new BytesRef("non_existent")));
@@ -249,11 +255,15 @@ public class SuggestFieldTest extends Lu
document.add(newStringField("delete", "delete", Field.Store.NO));
iw.addDocument(document);
document.clear();
+
+ if (usually()) {
+ iw.commit();
+ }
}
iw.deleteDocuments(new Term("delete", "delete"));
- DirectoryReader reader = DirectoryReader.open(iw, false);
+ DirectoryReader reader = DirectoryReader.open(iw, true);
SuggestIndexSearcher indexSearcher = new SuggestIndexSearcher(reader, analyzer);
TopSuggestDocs suggest = indexSearcher.suggest("suggest_field", "abc_", num);
assertThat(suggest.totalHits, equalTo(0));
@@ -274,6 +284,10 @@ public class SuggestFieldTest extends Lu
document.add(new IntField("weight_fld", i, Field.Store.YES));
iw.addDocument(document);
document.clear();
+
+ if (usually()) {
+ iw.commit();
+ }
}
iw.deleteDocuments(NumericRangeQuery.newIntRange("weight_fld", 2, null, true, false));
@@ -298,6 +312,10 @@ public class SuggestFieldTest extends Lu
document.add(new IntField("filter_int_fld", i, Field.Store.NO));
iw.addDocument(document);
document.clear();
+
+ if (usually()) {
+ iw.commit();
+ }
}
DirectoryReader reader = iw.getReader();
@@ -542,6 +560,10 @@ public class SuggestFieldTest extends Lu
document.add(newSuggestField("suggest_field", suggest, weight));
mappings.put(suggest, weight);
iw.addDocument(document);
+
+ if (usually()) {
+ iw.commit();
+ }
}
DirectoryReader reader = iw.getReader();
Modified: lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java (original)
+++ lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java Fri Apr 3 20:13:06 2015
@@ -263,6 +263,9 @@ public abstract class SearchEquivalenceT
* Both queries will be filtered by <code>filter</code>
*/
protected void assertSubsetOf(Query q1, Query q2, Filter filter) throws Exception {
+ QueryUtils.check(q1);
+ QueryUtils.check(q2);
+
if (filter != null) {
q1 = new FilteredQuery(q1, filter);
q2 = new FilteredQuery(q2, filter);
Modified: lucene/dev/branches/lucene6271/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/CHANGES.txt?rev=1671151&r1=1671150&r2=1671151&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6271/solr/CHANGES.txt Fri Apr 3 20:13:06 2015
@@ -78,6 +78,9 @@ Detailed Change List
New Features
----------------------
+* SOLR-6637: Solr should have a way to restore a core from a backed up index.
+ (Varun Thacker, noble, shalin)
+
Bug Fixes
----------------------
@@ -90,6 +93,11 @@ Optimizations
* SOLR-7324: IndexFetcher does not need to call isIndexStale if full copy is already needed
(Stephan Lagraulet via Varun Thacker)
+Other Changes
+----------------------
+
+* SOLR-6865: Upgrade HttpClient to 4.4.1 (Shawn Heisey)
+
================== 5.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release