You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/04/16 12:11:15 UTC
svn commit: r1587851 - in /lucene/dev/trunk/lucene:
core/src/java/org/apache/lucene/search/
core/src/test/org/apache/lucene/search/
suggest/src/java/org/apache/lucene/search/suggest/analyzing/
Author: mikemccand
Date: Wed Apr 16 10:11:14 2014
New Revision: 1587851
URL: http://svn.apache.org/r1587851
Log:
fix AnalyzingInfixSuggester's javadocs to make it clearer that the index is 'private' to the suggester and should not be the same index used for normal searching
Modified:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1587851&r1=1587850&r2=1587851&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Wed Apr 16 10:11:14 2014
@@ -313,6 +313,7 @@ public class BooleanQuery extends Query
return super.bulkScorer(context, scoreDocsInOrder, acceptDocs);
}
+ List<Scorer> required = new ArrayList<Scorer>();
List<BulkScorer> prohibited = new ArrayList<BulkScorer>();
List<BulkScorer> optional = new ArrayList<BulkScorer>();
Iterator<BooleanClause> cIter = clauses.iterator();
@@ -327,7 +328,11 @@ public class BooleanQuery extends Query
// TODO: there are some cases where BooleanScorer
// would handle conjunctions faster than
// BooleanScorer2...
- return super.bulkScorer(context, scoreDocsInOrder, acceptDocs);
+// return super.bulkScorer(context, scoreDocsInOrder, acceptDocs);
+ Scorer requiredSubScorer = w.scorer(context, acceptDocs);
+ if ( requiredSubScorer == null ) return null;
+ required.add(requiredSubScorer);
+
} else if (c.isProhibited()) {
prohibited.add(subScorer);
} else {
@@ -336,7 +341,7 @@ public class BooleanQuery extends Query
}
// Check if we can and should return a BooleanScorer
- return new BooleanScorer(this, disableCoord, minNrShouldMatch, optional, prohibited, maxCoord);
+ return new BooleanScorer(this, disableCoord, minNrShouldMatch, required, optional, prohibited, maxCoord);
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java?rev=1587851&r1=1587850&r2=1587851&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java Wed Apr 16 10:11:14 2014
@@ -61,7 +61,7 @@ import org.apache.lucene.search.BooleanQ
final class BooleanScorer extends BulkScorer {
- private static final class BooleanScorerCollector extends SimpleCollector {
+ private /*static*/ final class BooleanScorerCollector extends SimpleCollector {
private BucketTable bucketTable;
private int mask;
private Scorer scorer;
@@ -77,18 +77,19 @@ final class BooleanScorer extends BulkSc
final int i = doc & BucketTable.MASK;
final Bucket bucket = table.buckets[i];
+ final int coord = (mask & REQUIRED_MASK) == REQUIRED_MASK ? requiredNrMatchers : 1;
if (bucket.doc != doc) { // invalid bucket
bucket.doc = doc; // set doc
bucket.score = scorer.score(); // initialize score
bucket.bits = mask; // initialize mask
- bucket.coord = 1; // initialize coord
+ bucket.coord = /*1*/coord; // initialize coord
bucket.next = table.first; // push onto valid list
table.first = bucket;
} else { // valid bucket
bucket.score += scorer.score(); // increment score
bucket.bits |= mask; // add bits in mask
- bucket.coord++; // increment coord
+ bucket.coord/*++*/+= coord; // increment coord
}
}
@@ -116,7 +117,7 @@ final class BooleanScorer extends BulkSc
}
/** A simple hash table of document scores within a range. */
- static final class BucketTable {
+ /*static*/ final class BucketTable {
public static final int SIZE = 1 << 11;
public static final int MASK = SIZE - 1;
@@ -140,8 +141,7 @@ final class BooleanScorer extends BulkSc
static final class SubScorer {
public BulkScorer scorer;
- // TODO: re-enable this if BQ ever sends us required clauses
- //public boolean required = false;
+ public boolean required = false;
public boolean prohibited;
public LeafCollector collector;
public SubScorer next;
@@ -149,13 +149,9 @@ final class BooleanScorer extends BulkSc
public SubScorer(BulkScorer scorer, boolean required, boolean prohibited,
LeafCollector collector, SubScorer next) {
- if (required) {
- throw new IllegalArgumentException("this scorer cannot handle required=true");
- }
this.scorer = scorer;
this.more = true;
- // TODO: re-enable this if BQ ever sends us required clauses
- //this.required = required;
+ this.required = required;
this.prohibited = prohibited;
this.collector = collector;
this.next = next;
@@ -165,20 +161,30 @@ final class BooleanScorer extends BulkSc
private SubScorer scorers = null;
private BucketTable bucketTable = new BucketTable();
private final float[] coordFactors;
- // TODO: re-enable this if BQ ever sends us required clauses
- //private int requiredMask = 0;
private final int minNrShouldMatch;
private int end;
private Bucket current;
// Any time a prohibited clause matches we set bit 0:
private static final int PROHIBITED_MASK = 1;
+ // Any time a prohibited clause matches we set bit 1:
+ private static final int REQUIRED_MASK = 2;
+ private final int requiredNrMatchers;
private final Weight weight;
- BooleanScorer(BooleanWeight weight, boolean disableCoord, int minNrShouldMatch,
- List<BulkScorer> optionalScorers, List<BulkScorer> prohibitedScorers, int maxCoord) throws IOException {
+ BooleanScorer(BooleanWeight weight, boolean disableCoord, int minNrShouldMatch,
+ List<Scorer> requiredScorers, List<BulkScorer> optionalScorers, List<BulkScorer> prohibitedScorers,
+ int maxCoord) throws IOException {
+
this.minNrShouldMatch = minNrShouldMatch;
this.weight = weight;
+
+ this.requiredNrMatchers = requiredScorers.size();
+ if ( this.requiredNrMatchers > 0 ) {
+ BulkScorer requiredScorer = new Weight.DefaultBulkScorer(new ConjunctionScorer(
+ this.weight, requiredScorers.toArray(new Scorer[requiredScorers.size()])));
+ scorers = new SubScorer(requiredScorer, true, false, bucketTable.newCollector(REQUIRED_MASK), scorers);
+ }
for (BulkScorer scorer : optionalScorers) {
scorers = new SubScorer(scorer, false, false, bucketTable.newCollector(0), scorers);
@@ -188,7 +194,8 @@ final class BooleanScorer extends BulkSc
scorers = new SubScorer(scorer, false, true, bucketTable.newCollector(PROHIBITED_MASK), scorers);
}
- coordFactors = new float[optionalScorers.size() + 1];
+ // TODO: required add requriredScorer.size().
+ coordFactors = new float[requiredScorers.size() + optionalScorers.size() + 1];
for (int i = 0; i < coordFactors.length; i++) {
coordFactors[i] = disableCoord ? 1.0f : weight.coord(i, maxCoord);
}
@@ -209,12 +216,9 @@ final class BooleanScorer extends BulkSc
while (current != null) { // more queued
// check prohibited & required
- if ((current.bits & PROHIBITED_MASK) == 0) {
+ if ((current.bits & PROHIBITED_MASK) == 0 &&
+ (requiredNrMatchers == 0 || (current.bits & REQUIRED_MASK) == REQUIRED_MASK)) {
- // TODO: re-enable this if BQ ever sends us required
- // clauses
- //&& (current.bits & requiredMask) == requiredMask) {
-
// NOTE: Lucene always passes max =
// Integer.MAX_VALUE today, because we never embed
// a BooleanScorer inside another (even though
@@ -229,7 +233,7 @@ final class BooleanScorer extends BulkSc
continue;
}
- if (current.coord >= minNrShouldMatch) {
+ if (current.coord >= minNrShouldMatch + requiredNrMatchers) {
fs.score = (float) (current.score * coordFactors[current.coord]);
fs.doc = current.doc;
fs.freq = current.coord;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=1587851&r1=1587850&r2=1587851&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Wed Apr 16 10:11:14 2014
@@ -77,6 +77,8 @@ public class MultiTermQueryWrapperFilter
/** Returns the field name for this query */
public final String getField() { return query.getField(); }
+
+ public static long rewriteTermCount;
/**
* Returns a DocIdSet with documents that should be permitted in search
@@ -104,6 +106,7 @@ public class MultiTermQueryWrapperFilter
final FixedBitSet bitSet = new FixedBitSet(context.reader().maxDoc());
DocsEnum docsEnum = null;
do {
+ rewriteTermCount++;
// System.out.println(" iter termCount=" + termCount + " term=" +
// enumerator.term().toBytesString());
docsEnum = termsEnum.docs(acceptDocs, docsEnum, DocsEnum.FLAG_NONE);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java?rev=1587851&r1=1587850&r2=1587851&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java Wed Apr 16 10:11:14 2014
@@ -96,7 +96,7 @@ public class TestBooleanScorer extends L
}
}};
- BooleanScorer bs = new BooleanScorer(weight, false, 1, Arrays.asList(scorers), Collections.<BulkScorer>emptyList(), scorers.length);
+ BooleanScorer bs = new BooleanScorer(weight, false, 1, Collections.<Scorer>emptyList(), Arrays.asList(scorers), Collections.<BulkScorer>emptyList(), scorers.length);
final List<Integer> hits = new ArrayList<>();
bs.score(new SimpleCollector() {
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1587851&r1=1587850&r2=1587851&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Wed Apr 16 10:11:14 2014
@@ -93,17 +93,16 @@ import org.apache.lucene.util.Version;
* on prefix matches to any tokens in the indexed text.
* This also highlights the tokens that match.
*
- * <p>This just uses an ordinary Lucene index. It
- * supports payloads, and records these as a
- * {@link BinaryDocValues} field. Matches are sorted only
+ * <p>This suggester supports payloads. Matches are sorted only
* by the suggest weight; it would be nice to support
* blended score + weight sort in the future. This means
* this suggester best applies when there is a strong
- * apriori ranking of all the suggestions.
+ * a-priori ranking of all the suggestions.
*
* <p>This suggester supports contexts, however the
* contexts must be valid utf8 (arbitrary binary terms will
* not work).
+ *
* @lucene.experimental */
public class AnalyzingInfixSuggester extends Lookup implements Closeable {
@@ -141,14 +140,18 @@ public class AnalyzingInfixSuggester ext
private static final Sort SORT = new Sort(new SortField("weight", SortField.Type.LONG, true));
/** Create a new instance, loading from a previously built
- * directory, if it exists. Note that {@link #close}
+ * AnalyzingInfixSuggester directory, if it exists. This directory must be
+ * private to the infix suggester (i.e., not an external
+ * Lucene index). Note that {@link #close}
* will also close the provided directory. */
public AnalyzingInfixSuggester(Version matchVersion, Directory dir, Analyzer analyzer) throws IOException {
this(matchVersion, dir, analyzer, analyzer, DEFAULT_MIN_PREFIX_CHARS);
}
/** Create a new instance, loading from a previously built
- * directory, if it exists. Note that {@link #close}
+ * AnalyzingInfixSuggester directory, if it exists. This directory must be
+ * private to the infix suggester (i.e., not an external
+ * Lucene index). Note that {@link #close}
* will also close the provided directory.
*
* @param minPrefixChars Minimum number of leading characters