You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2016/11/22 17:54:01 UTC
lucene-solr:branch_6x: SOLR-9786: FieldType.getSetQuery,
QParser.flags/isFilter,
solr query parser use getSetQuery when appropriate and other optimizations
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 615570c34 -> 9f24c4b59
SOLR-9786: FieldType.getSetQuery, QParser.flags/isFilter, solr query parser use getSetQuery when appropriate and other optimizations
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9f24c4b5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9f24c4b5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9f24c4b5
Branch: refs/heads/branch_6x
Commit: 9f24c4b59a89bb1d42b2e93abbcfb4076777e321
Parents: 615570c
Author: yonik <yo...@apache.org>
Authored: Tue Nov 22 12:32:53 2016 -0500
Committer: yonik <yo...@apache.org>
Committed: Tue Nov 22 12:53:41 2016 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 10 +
.../solr/handler/component/QueryComponent.java | 3 +-
.../org/apache/solr/parser/QueryParser.java | 11 +-
.../java/org/apache/solr/parser/QueryParser.jj | 7 +-
.../apache/solr/parser/SolrQueryParserBase.java | 194 +++++++++++++++---
.../java/org/apache/solr/schema/FieldType.java | 28 ++-
.../solr/search/ExtendedDismaxQParser.java | 6 +-
.../solr/search/FunctionRangeQParserPlugin.java | 4 +-
.../java/org/apache/solr/search/QParser.java | 29 +++
.../org/apache/solr/util/SolrPluginUtils.java | 6 +-
.../solr/search/TestExtendedDismaxParser.java | 4 +-
.../apache/solr/search/TestSolrQueryParser.java | 198 ++++++++++++++++++-
12 files changed, 456 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index db590b4..3c10127 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -88,6 +88,16 @@ Optimizations
* SOLR-9772: Deriving distributed sort values (fieldSortValues) should reuse
comparator and only invalidate leafComparator. (John Call via yonik)
+* SOLR-9786: FieldType has a new getSetQuery() method that can take a set of terms
+ and create a more efficient query (such as TermsQuery). The solr query parser has been
+ changed to use this method when appropriate. The parser also knows when it is being
+ used to parse a filter and will create TermsQueries from large lists of normal terms
+ or numbers, resulting in a query that will execute faster. This also acts to avoid
+ BooleanQuery maximum clause limit. Query parsing itself has also been optimized,
+ resulting in less produced garbage and 5-7% better performance.
+ (yonik)
+
+
Bug Fixes
----------------------
* SOLR-9701: NPE in export handler when "fl" parameter is omitted.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index 1e54262..e9c9433 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -200,10 +200,11 @@ public class QueryComponent extends SearchComponent
if (fqs!=null && fqs.length!=0) {
List<Query> filters = rb.getFilters();
// if filters already exists, make a copy instead of modifying the original
- filters = filters == null ? new ArrayList<Query>(fqs.length) : new ArrayList<>(filters);
+ filters = filters == null ? new ArrayList<>(fqs.length) : new ArrayList<>(filters);
for (String fq : fqs) {
if (fq != null && fq.trim().length()!=0) {
QParser fqp = QParser.getParser(fq, req);
+ fqp.setIsFilter(true);
filters.add(fqp.getQuery());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/parser/QueryParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/parser/QueryParser.java b/solr/core/src/java/org/apache/solr/parser/QueryParser.java
index b2ef500..42f359e 100644
--- a/solr/core/src/java/org/apache/solr/parser/QueryParser.java
+++ b/solr/core/src/java/org/apache/solr/parser/QueryParser.java
@@ -4,11 +4,12 @@ package org.apache.solr.parser;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
+
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Version;
-import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.QParser;
+import org.apache.solr.search.SyntaxError;
public class QueryParser extends SolrQueryParserBase implements QueryParserConstants {
@@ -135,9 +136,9 @@ public class QueryParser extends SolrQueryParserBase implements QueryParserConst
addClause(clauses, conj, mods, q);
}
if (clauses.size() == 1 && firstQuery != null)
- {if (true) return firstQuery;}
+ {if (true) return rawToNormal(firstQuery);}
else {
- {if (true) return getBooleanQuery(clauses);}
+ {if (true) return getBooleanQuery(clauses);}
}
throw new Error("Missing return statement in function");
}
@@ -146,6 +147,7 @@ public class QueryParser extends SolrQueryParserBase implements QueryParserConst
Query q;
Token fieldToken=null, boost=null;
Token localParams=null;
+ int flags = 0;
if (jj_2_1(2)) {
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case TERM:
@@ -195,6 +197,7 @@ public class QueryParser extends SolrQueryParserBase implements QueryParserConst
break;
case FILTER:
jj_consume_token(FILTER);
+ flags=startFilter();
q = Query(field);
jj_consume_token(RPAREN);
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
@@ -206,7 +209,7 @@ public class QueryParser extends SolrQueryParserBase implements QueryParserConst
jj_la1[7] = jj_gen;
;
}
- q=getFilter(q);
+ q=getFilter(q); restoreFlags(flags);
break;
case LPARAMS:
localParams = jj_consume_token(LPARAMS);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/parser/QueryParser.jj
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/parser/QueryParser.jj b/solr/core/src/java/org/apache/solr/parser/QueryParser.jj
index c5752f6..a6b93ca 100644
--- a/solr/core/src/java/org/apache/solr/parser/QueryParser.jj
+++ b/solr/core/src/java/org/apache/solr/parser/QueryParser.jj
@@ -190,9 +190,9 @@ Query Query(String field) throws SyntaxError :
)*
{
if (clauses.size() == 1 && firstQuery != null)
- return firstQuery;
+ return rawToNormal(firstQuery);
else {
- return getBooleanQuery(clauses);
+ return getBooleanQuery(clauses);
}
}
}
@@ -201,6 +201,7 @@ Query Clause(String field) throws SyntaxError : {
Query q;
Token fieldToken=null, boost=null;
Token localParams=null;
+ int flags = 0;
}
{
@@ -216,7 +217,7 @@ Query Clause(String field) throws SyntaxError : {
(
q=Term(field)
| <LPAREN> q=Query(field) <RPAREN> (<CARAT> boost=<NUMBER>)?
- | (<FILTER> q=Query(field) <RPAREN> (<CARAT> boost=<NUMBER>)? { q=getFilter(q); } )
+ | (<FILTER> { flags=startFilter(); } q=Query(field) <RPAREN> (<CARAT> boost=<NUMBER>)? { q=getFilter(q); restoreFlags(flags); } )
| (localParams = <LPARAMS> (<CARAT> boost=<NUMBER>)? { q=getLocalParams(field, localParams.image); } )
)
{ return handleBoost(q, boost); }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
index 24a6f3e..0f6c7ac 100644
--- a/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
+++ b/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
@@ -17,6 +17,7 @@
package org.apache.solr.parser;
import java.io.StringReader;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
@@ -61,6 +62,7 @@ import org.apache.solr.search.SyntaxError;
*/
public abstract class SolrQueryParserBase extends QueryBuilder {
+ public static final int TERMS_QUERY_THRESHOLD = 16; // @lucene.internal Set to a low value temporarily for better test coverage
static final int CONJ_NONE = 0;
static final int CONJ_AND = 1;
@@ -89,7 +91,7 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
int fuzzyPrefixLength = FuzzyQuery.defaultPrefixLength;
boolean autoGeneratePhraseQueries = false;
-
+ int flags;
protected IndexSchema schema;
protected QParser parser;
@@ -125,6 +127,31 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
}
}
+ // internal: A simple raw fielded query
+ public static class RawQuery extends Query {
+ final SchemaField sfield;
+ final String externalVal;
+
+ public RawQuery(SchemaField sfield, String externalVal) {
+ this.sfield = sfield;
+ this.externalVal = externalVal;
+ }
+
+ @Override
+ public String toString(String field) {
+ return "RAW(" + field + "," + externalVal + ")";
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+ }
// So the generated QueryParser(CharStream) won't error out
protected SolrQueryParserBase() {
@@ -138,10 +165,22 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
public void init(Version matchVersion, String defaultField, QParser parser) {
this.schema = parser.getReq().getSchema();
this.parser = parser;
+ this.flags = parser.getFlags();
this.defaultField = defaultField;
setAnalyzer(schema.getQueryAnalyzer());
}
+ // Turn on the "filter" bit and return the previous flags for the caller to save
+ int startFilter() {
+ int oldFlags = flags;
+ flags |= QParser.FLAG_FILTER;
+ return oldFlags;
+ }
+
+ void restoreFlags(int flagsToRestore) {
+ flags = flagsToRestore;
+ }
+
/** Parses a query string, returning a {@link org.apache.lucene.search.Query}.
* @param query the query string to be parsed.
*/
@@ -381,7 +420,7 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
*/
protected Query getFieldQuery(String field, String queryText, int slop)
throws SyntaxError {
- Query query = getFieldQuery(field, queryText, true);
+ Query query = getFieldQuery(field, queryText, true, false);
// only set slop of the phrase query was a result of this parser
// and not a sub-parser.
@@ -492,11 +531,77 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
if (clauses.size()==0) {
return null; // all clause words were filtered away by the analyzer.
}
- BooleanQuery.Builder query = newBooleanQuery();
- for(final BooleanClause clause: clauses) {
- query.add(clause);
+
+ SchemaField sfield = null;
+ List<String> fieldValues = null;
+
+
+ boolean useTermsQuery = (flags & QParser.FLAG_FILTER)!=0 && clauses.size() > TERMS_QUERY_THRESHOLD;
+ int clausesAdded = 0;
+
+ BooleanQuery.Builder booleanBuilder = newBooleanQuery();
+ Map<SchemaField, List<String>> fmap = new HashMap<>();
+
+ for (BooleanClause clause : clauses) {
+ Query subq = clause.getQuery();
+ if (subq instanceof RawQuery) {
+ if (clause.getOccur() != BooleanClause.Occur.SHOULD) {
+ // We only collect optional terms for set queries. Since this isn't optional,
+ // convert the raw query to a normal query and handle as usual.
+ clause = new BooleanClause( rawToNormal(subq), clause.getOccur() );
+ } else {
+ // Optional raw query.
+ RawQuery rawq = (RawQuery) subq;
+
+ // only look up fmap and type info on a field change
+ if (sfield != rawq.sfield) {
+ sfield = rawq.sfield;
+ fieldValues = fmap.get(sfield);
+ // If this field isn't indexed, or if it is indexed and we want to use TermsQuery, then collect this value.
+ // We are currently relying on things like PointField not being marked as indexed in order to bypass
+ // the "useTermQuery" check.
+ if (fieldValues == null && useTermsQuery || !sfield.indexed()) {
+ fieldValues = new ArrayList<>(2);
+ fmap.put(sfield, fieldValues);
+ }
+ }
+
+ if (fieldValues != null) {
+ fieldValues.add(rawq.externalVal);
+ continue;
+ }
+
+ clause = new BooleanClause( rawToNormal(subq), clause.getOccur() );
+ }
+ }
+
+ clausesAdded++;
+ booleanBuilder.add(clause);
+ }
+
+
+ for (Map.Entry<SchemaField,List<String>> entry : fmap.entrySet()) {
+ sfield = entry.getKey();
+ fieldValues = entry.getValue();
+ FieldType ft = sfield.getType();
+
+ // TODO: pull more of this logic out to FieldType? We would need to be able to add clauses to our existing booleanBuilder.
+ if (sfield.indexed() && fieldValues.size() < TERMS_QUERY_THRESHOLD || fieldValues.size() == 1) {
+ // use boolean query instead
+ for (String externalVal : fieldValues) {
+ Query subq = ft.getFieldQuery(this.parser, sfield, externalVal);
+ clausesAdded++;
+ booleanBuilder.add(subq, BooleanClause.Occur.SHOULD);
+ }
+ } else {
+ Query subq = ft.getSetQuery(this.parser, sfield, fieldValues);
+ if (fieldValues.size() == clauses.size()) return subq; // if this is everything, don't wrap in a boolean query
+ clausesAdded++;
+ booleanBuilder.add(subq, BooleanClause.Occur.SHOULD);
+ }
}
- return query.build();
+
+ return booleanBuilder.build();
}
@@ -526,7 +631,7 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
q = getFuzzyQuery(qfield, termImage, fms);
} else {
String termImage=discardEscapeChar(term.image);
- q = getFieldQuery(qfield, termImage, false);
+ q = getFieldQuery(qfield, termImage, false, true);
}
return q;
}
@@ -540,10 +645,15 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
}
catch (Exception ignored) { }
}
- return getFieldQuery(qfield, discardEscapeChar(term.image.substring(1, term.image.length()-1)), s);
+
+ String raw = discardEscapeChar(term.image.substring(1, term.image.length()-1));
+ return getFieldQuery(qfield, raw, s);
}
- // called from parser
+
+
+ // Called from parser
+ // Raw queries are transformed to normal queries before wrapping in a BoostQuery
Query handleBoost(Query q, Token boost) {
// q==null check is to avoid boosting null queries, such as those caused by stop words
if (boost == null || boost.image.length()==0 || q == null) {
@@ -556,14 +666,14 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
if (q instanceof ConstantScoreQuery || q instanceof SolrConstantScoreQuery) {
// skip
} else {
- newQ = new ConstantScoreQuery(q);
+ newQ = new ConstantScoreQuery( rawToNormal(q) );
}
return new BoostQuery(newQ, val);
}
float boostVal = Float.parseFloat(boost.image);
- return new BoostQuery(q, boostVal);
+ return new BoostQuery( rawToNormal(q), boostVal);
}
@@ -577,17 +687,21 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
*
*/
String discardEscapeChar(String input) throws SyntaxError {
+ int start = input.indexOf('\\');
+ if (start < 0) return input;
+
// Create char array to hold unescaped char sequence
char[] output = new char[input.length()];
+ input.getChars(0, start, output, 0);
// The length of the output can be less than the input
// due to discarded escape chars. This variable holds
// the actual length of the output
- int length = 0;
+ int length = start;
// We remember whether the last processed character was
// an escape character
- boolean lastCharWasEscapeChar = false;
+ boolean lastCharWasEscapeChar = true;
// The multiplier the current unicode digit must be multiplied with.
// E. g. the first digit must be multiplied with 16^3, the second with 16^2...
@@ -596,7 +710,8 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
// Used to calculate the codepoint of the escaped unicode character
int codePoint = 0;
- for (int i = 0; i < input.length(); i++) {
+ // start after the first escape char
+ for (int i = start+1; i < input.length(); i++) {
char curChar = input.charAt(i);
if (codePointMultiplier > 0) {
codePoint += hexToInt(curChar) * codePointMultiplier;
@@ -715,25 +830,57 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
private QParser subQParser = null;
+ // Create a "normal" query from a RawQuery (or just return the current query if it's not raw)
+ Query rawToNormal(Query q) {
+ if (!(q instanceof RawQuery)) return q;
+ RawQuery rq = (RawQuery)q;
+ return rq.sfield.getType().getFieldQuery(parser, rq.sfield, rq.externalVal);
+ }
+
protected Query getFieldQuery(String field, String queryText, boolean quoted) throws SyntaxError {
+ return getFieldQuery(field, queryText, quoted, false);
+ }
+
+ // private use for getFieldQuery
+ private String lastFieldName;
+ private SchemaField lastField;
+
+ // if raw==true, then it's possible for this method to return a RawQuery that will need to be transformed
+ // further before using.
+ protected Query getFieldQuery(String field, String queryText, boolean quoted, boolean raw) throws SyntaxError {
checkNullField(field);
- // intercept magic field name of "_" to use as a hook for our
- // own functions.
- if (field.charAt(0) == '_' && parser != null) {
- MagicFieldName magic = MagicFieldName.get(field);
- if (null != magic) {
- subQParser = parser.subQuery(queryText, magic.subParser);
- return subQParser.getQuery();
+
+ SchemaField sf;
+ if (field.equals(lastFieldName)) {
+ // only look up the SchemaField on a field change... this helps with memory allocation of dynamic fields
+ // and large queries like foo_i:(1 2 3 4 5 6 7 8 9 10) when we are passed "foo_i" each time.
+ sf = lastField;
+ } else {
+ // intercept magic field name of "_" to use as a hook for our
+ // own functions.
+ if (field.charAt(0) == '_' && parser != null) {
+ MagicFieldName magic = MagicFieldName.get(field);
+ if (null != magic) {
+ subQParser = parser.subQuery(queryText, magic.subParser);
+ return subQParser.getQuery();
+ }
}
+
+ lastFieldName = field;
+ sf = lastField = schema.getFieldOrNull(field);
}
- SchemaField sf = schema.getFieldOrNull(field);
+
if (sf != null) {
FieldType ft = sf.getType();
// delegate to type for everything except tokenized fields
if (ft.isTokenized() && sf.indexed()) {
return newFieldQuery(getAnalyzer(), field, queryText, quoted || (ft instanceof TextField && ((TextField)ft).getAutoGeneratePhraseQueries()));
} else {
- return sf.getType().getFieldQuery(parser, sf, queryText);
+ if (raw) {
+ return new RawQuery(sf, queryText);
+ } else {
+ return sf.getType().getFieldQuery(parser, sf, queryText);
+ }
}
}
@@ -742,6 +889,7 @@ public abstract class SolrQueryParserBase extends QueryBuilder {
}
+
// called from parser
protected Query getRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) throws SyntaxError {
checkNullField(field);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/schema/FieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index f3fdcba..bf94d8b 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -19,6 +19,7 @@ package org.apache.solr.schema;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -38,7 +39,10 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
+import org.apache.lucene.queries.TermsQuery;
import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DocValuesRangeQuery;
import org.apache.lucene.search.DocValuesRewriteMethod;
import org.apache.lucene.search.MultiTermQuery;
@@ -56,8 +60,8 @@ import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.Version;
import org.apache.solr.analysis.SolrAnalyzer;
import org.apache.solr.analysis.TokenizerChain;
-import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
@@ -758,7 +762,27 @@ public abstract class FieldType extends FieldProperties {
return new TermQuery(new Term(field.getName(), br));
}
}
-
+
+ /** @lucene.experimental */
+ public Query getSetQuery(QParser parser, SchemaField field, Collection<String> externalVals) {
+ if (!field.indexed()) {
+ BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ for (String externalVal : externalVals) {
+ Query subq = getFieldQuery(parser, field, externalVal);
+ builder.add(subq, BooleanClause.Occur.SHOULD);
+ }
+ return builder.build();
+ }
+
+ List<BytesRef> lst = new ArrayList<>(externalVals.size());
+ BytesRefBuilder br = new BytesRefBuilder();
+ for (String externalVal : externalVals) {
+ readableToIndexed(externalVal, br);
+ lst.add( br.toBytesRef() );
+ }
+ return new TermsQuery(field.getName() , lst);
+ }
+
/**
* Expert: Returns the rewrite method for multiterm queries such as wildcards.
* @param parser The {@link org.apache.solr.search.QParser} calling the method
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
index 563f8a0..9e2c4d7 100644
--- a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
@@ -1034,8 +1034,8 @@ public class ExtendedDismaxQParser extends QParser {
}
@Override
- protected Query getFieldQuery(String field, String val, boolean quoted) throws SyntaxError {
- this.type = QType.FIELD;
+ protected Query getFieldQuery(String field, String val, boolean quoted, boolean raw) throws SyntaxError {
+ this.type = quoted ? QType.PHRASE : QType.FIELD;
this.field = field;
this.val = val;
this.slop = getPhraseSlop(); // unspecified
@@ -1216,7 +1216,7 @@ public class ExtendedDismaxQParser extends QParser {
switch (type) {
case FIELD: // fallthrough
case PHRASE:
- Query query = super.getFieldQuery(field, val, type == QType.PHRASE);
+ Query query = super.getFieldQuery(field, val, type == QType.PHRASE, false);
// A BooleanQuery is only possible from getFieldQuery if it came from
// a single whitespace separated term. In this case, check the coordination
// factor on the query: if it's enabled, that means we aren't a set of synonyms
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
index 1b37a97..4cda8ee 100644
--- a/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/FunctionRangeQParserPlugin.java
@@ -46,7 +46,9 @@ public class FunctionRangeQParserPlugin extends QParserPlugin {
@Override
public Query parse() throws SyntaxError {
funcStr = localParams.get(QueryParsing.V, null);
- Query funcQ = subQuery(funcStr, FunctionQParserPlugin.NAME).getQuery();
+ QParser subParser = subQuery(funcStr, FunctionQParserPlugin.NAME);
+ subParser.setIsFilter(false); // the range can be based on the relevancy score of embedded queries.
+ Query funcQ = subParser.getQuery();
if (funcQ instanceof FunctionQuery) {
vs = ((FunctionQuery)funcQ).getValueSource();
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/search/QParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/QParser.java b/solr/core/src/java/org/apache/solr/search/QParser.java
index 40605be..58c0265 100644
--- a/solr/core/src/java/org/apache/solr/search/QParser.java
+++ b/solr/core/src/java/org/apache/solr/search/QParser.java
@@ -32,12 +32,18 @@ import java.util.*;
*
*/
public abstract class QParser {
+ /** @lucene.experimental */
+ public static final int FLAG_FILTER = 0x01;
+
protected String qstr;
protected SolrParams params;
protected SolrParams localParams;
protected SolrQueryRequest req;
protected int recurseCount;
+ /** @lucene.experimental */
+ protected int flags;
+
protected Query query;
protected String stringIncludingLocalParams; // the original query string including any local params
@@ -83,6 +89,28 @@ public abstract class QParser {
this.req = req;
}
+ /** @lucene.experimental */
+ public void setFlags(int flags) {
+ this.flags = flags;
+ }
+
+ /** @lucene.experimental */
+ public int getFlags() {
+ return flags;
+ }
+
+ /** @lucene.experimental Query is in the context of a filter, where scores don't matter */
+ public boolean isFilter() {
+ return (flags & FLAG_FILTER) != 0;
+ }
+
+ /** @lucene.experimental */
+ public void setIsFilter(boolean isFilter) {
+ if (isFilter)
+ flags |= FLAG_FILTER;
+ else
+ flags &= ~FLAG_FILTER;
+ }
private static void addTag(Map<Object,Collection<Object>> tagMap, Object key, Object val) {
Collection<Object> lst = tagMap.get(key);
@@ -201,6 +229,7 @@ public abstract class QParser {
defaultType = localParams.get(QueryParsing.DEFTYPE);
}
QParser nestedParser = getParser(q, defaultType, getReq());
+ nestedParser.flags = this.flags; // TODO: this would be better passed in to the constructor... change to a ParserContext object?
nestedParser.recurseCount = recurseCount;
recurseCount--;
return nestedParser;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
index 638e30e..efba8e3 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrPluginUtils.java
@@ -908,7 +908,7 @@ public class SolrPluginUtils {
* aliases should work)
*/
@Override
- protected Query getFieldQuery(String field, String queryText, boolean quoted)
+ protected Query getFieldQuery(String field, String queryText, boolean quoted, boolean raw)
throws SyntaxError {
if (aliases.containsKey(field)) {
@@ -918,7 +918,7 @@ public class SolrPluginUtils {
List<Query> disjuncts = new ArrayList<>();
for (String f : a.fields.keySet()) {
- Query sub = getFieldQuery(f,queryText,quoted);
+ Query sub = getFieldQuery(f,queryText,quoted, false);
if (null != sub) {
if (null != a.fields.get(f)) {
sub = new BoostQuery(sub, a.fields.get(f));
@@ -932,7 +932,7 @@ public class SolrPluginUtils {
} else {
try {
- return super.getFieldQuery(field, queryText, quoted);
+ return super.getFieldQuery(field, queryText, quoted, raw);
} catch (Exception e) {
return null;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java b/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
index 9b305f2..c3b119f 100644
--- a/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
+++ b/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
@@ -1453,11 +1453,11 @@ public class TestExtendedDismaxParser extends SolrTestCaseJ4 {
@Override
protected Query getFieldQuery(String field,
- String val, boolean quoted) throws SyntaxError {
+ String val, boolean quoted, boolean raw) throws SyntaxError {
if(frequentlyMisspelledWords.contains(val)) {
return getFuzzyQuery(field, val, 0.75F);
}
- return super.getFieldQuery(field, val, quoted);
+ return super.getFieldQuery(field, val, quoted, raw);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f24c4b5/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
index 0b9c0f0..d3e6a7f 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
@@ -16,11 +16,20 @@
*/
package org.apache.solr.search;
+import java.util.Locale;
+import java.util.Random;
+
+import org.apache.lucene.queries.TermsQuery;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.core.SolrInfoMBean;
+import org.apache.solr.parser.QueryParser;
+import org.apache.solr.query.FilterQuery;
import org.apache.solr.request.SolrQueryRequest;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -37,9 +46,9 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
public static void createIndex() {
String v;
v = "how now brown cow";
- assertU(adoc("id", "1", "text", v, "text_np", v));
+ assertU(adoc("id", "1", "text", v, "text_np", v, "foo_i","11"));
v = "now cow";
- assertU(adoc("id", "2", "text", v, "text_np", v));
+ assertU(adoc("id", "2", "text", v, "text_np", v, "foo_i","12"));
assertU(adoc("id", "3", "foo_s", "a ' \" \\ {! ) } ( { z")); // A value filled with special chars
assertU(adoc("id", "10", "qqq_s", "X"));
@@ -184,6 +193,92 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
req.close();
}
+
+ // automatically use TermsQuery when appropriate
+ @Test
+ public void testAutoTerms() throws Exception {
+ SolrQueryRequest req = req();
+ QParser qParser;
+ Query q,qq;
+
+ // relevance query should not be a filter
+ qParser = QParser.getParser("foo_s:(a b c)", req);
+ q = qParser.getQuery();
+ assertEquals(3, ((BooleanQuery)q).clauses().size());
+
+ // small filter query should still use BooleanQuery
+ if (QueryParser.TERMS_QUERY_THRESHOLD > 3) {
+ qParser = QParser.getParser("foo_s:(a b c)", req);
+ qParser.setIsFilter(true); // this may change in the future
+ q = qParser.getQuery();
+ assertEquals(3, ((BooleanQuery) q).clauses().size());
+ }
+
+ // large relevancy query should use BooleanQuery
+ // TODO: we may decide that string fields shouldn't have relevance in the future... change to a text field w/o a stop filter if so
+ qParser = QParser.getParser("foo_s:(a b c d e f g h i j k l m n o p q r s t u v w x y z)", req);
+ q = qParser.getQuery();
+ assertEquals(26, ((BooleanQuery)q).clauses().size());
+
+ // large filter query should use TermsQuery
+ qParser = QParser.getParser("foo_s:(a b c d e f g h i j k l m n o p q r s t u v w x y z)", req);
+ qParser.setIsFilter(true); // this may change in the future
+ q = qParser.getQuery();
+ assertEquals(26, ((TermsQuery)q).getTermData().size());
+
+ // large numeric filter query should use TermsQuery (for trie fields)
+ qParser = QParser.getParser("foo_i:(1 2 3 4 5 6 7 8 9 10 20 19 18 17 16 15 14 13 12 11)", req);
+ qParser.setIsFilter(true); // this may change in the future
+ q = qParser.getQuery();
+ assertEquals(20, ((TermsQuery)q).getTermData().size());
+
+ // a filter() clause inside a relevancy query should be able to use a TermsQuery
+ qParser = QParser.getParser("foo_s:aaa filter(foo_s:(a b c d e f g h i j k l m n o p q r s t u v w x y z))", req);
+ q = qParser.getQuery();
+ assertEquals(2, ((BooleanQuery)q).clauses().size());
+ qq = ((BooleanQuery)q).clauses().get(0).getQuery();
+ if (qq instanceof TermQuery) {
+ qq = ((BooleanQuery)q).clauses().get(1).getQuery();
+ }
+
+ if (qq instanceof FilterQuery) {
+ qq = ((FilterQuery)qq).getQuery();
+ }
+
+ assertEquals(26, ((TermsQuery)qq).getTermData().size());
+
+ // test mixed boolean query, including quotes (which shouldn't matter)
+ qParser = QParser.getParser("foo_s:(a +aaa b -bbb c d e f bar_s:(qqq www) g h i j k l m n o p q r s t u v w x y z)", req);
+ qParser.setIsFilter(true); // this may change in the future
+ q = qParser.getQuery();
+ assertEquals(4, ((BooleanQuery)q).clauses().size());
+ qq = null;
+ for (BooleanClause clause : ((BooleanQuery)q).clauses()) {
+ qq = clause.getQuery();
+ if (qq instanceof TermsQuery) break;
+ }
+ assertEquals(26, ((TermsQuery)qq).getTermData().size());
+
+ req.close();
+ }
+
+ @Test
+ public void testManyClauses() throws Exception {
+ String a = "1 a 2 b 3 c 10 d 11 12 "; // 10 terms
+ StringBuilder sb = new StringBuilder("id:(");
+ for (int i = 0; i < 1024; i++) { // historically, the max number of boolean clauses defaulted to 1024
+ sb.append('z').append(i).append(' ');
+ }
+ sb.append(a);
+ sb.append(")");
+
+ String q = sb.toString();
+
+ // This will still fail when used as the main query, but will pass in a filter query since TermsQuery can be used.
+ assertJQ(req("q","*:*", "fq", q)
+ ,"/response/numFound==6");
+ }
+
@Test
public void testComments() throws Exception {
assertJQ(req("q", "id:1 id:2 /* *:* */ id:3")
@@ -317,4 +412,103 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
}
+ // parsing performance test
+ // Run from command line with ant test -Dtestcase=TestSolrQueryParser -Dtestmethod=testParsingPerformance -Dtests.asserts=false 2>/dev/null | grep QPS
+ @Test
+ public void testParsingPerformance() throws Exception {
+ String[] args = {"-queries","100" ,"-iter","1000", "-clauses","100", "-format","term%d", "-seed","0"};
+ args = new String[] {"-queries","1000" ,"-iter","2000", "-clauses","10", "-format","term%d", "-seed","0"};
+ // args = new String[] {"-queries","1000" ,"-iter","1000000000", "-clauses","10", "-format","term%d", "-seed","0"};
+
+ boolean assertOn = false;
+ assert assertOn = true;
+ if (assertOn) {
+ // System.out.println("WARNING! Assertions are enabled!!!! Will only execute small run. Change with -Dtests.asserts=false");
+ args = new String[]{"-queries","10" ,"-iter","2", "-clauses","20", "-format","term%d", "-seed","0"};
+ }
+
+
+ int iter = 1000;
+ int numQueries = 100;
+ int maxClauses = 5;
+ int maxTerm = 10000000;
+ String format = "term%d";
+ String field = "foo_s";
+ long seed = 0;
+ boolean isFilter = true;
+ boolean rewrite = false;
+
+ String otherStuff = "";
+
+ for (int i = 0; i < args.length; i++) {
+ String a = args[i];
+ if ("-queries".equals(a)) {
+ numQueries = Integer.parseInt(args[++i]);
+ } else if ("-iter".equals(a)) {
+ iter = Integer.parseInt(args[++i]);
+ } else if ("-clauses".equals(a)) {
+ maxClauses = Integer.parseInt(args[++i]);
+ } else if ("-format".equals(a)) {
+ format = args[++i];
+ } else if ("-seed".equals(a)) {
+ seed = Long.parseLong(args[++i]);
+ } else {
+ otherStuff = otherStuff + " " + a;
+ }
+ }
+
+ Random r = new Random(seed);
+
+ String[] queries = new String[numQueries];
+ for (int i = 0; i < queries.length; i++) {
+ StringBuilder sb = new StringBuilder();
+ boolean explicitField = r.nextInt(5) == 0;
+ if (!explicitField) {
+ sb.append(field + ":(");
+ }
+
+ sb.append(otherStuff).append(" ");
+
+ int nClauses = r.nextInt(maxClauses) + 1; // TODO: query parse can't parse () for some reason???
+
+ for (int c = 0; c<nClauses; c++) {
+ String termString = String.format(Locale.US, format, r.nextInt(maxTerm));
+ if (explicitField) {
+ sb.append(field).append(':');
+ }
+ sb.append(termString);
+ sb.append(' ');
+ }
+
+ if (!explicitField) {
+ sb.append(")");
+ }
+ queries[i] = sb.toString();
+ // System.out.println(queries[i]);
+ }
+
+ SolrQueryRequest req = req();
+
+ long start = System.nanoTime();
+
+ int ret = 0;
+ for (int i=0; i<iter; i++) {
+ for (String qStr : queries) {
+ QParser parser = QParser.getParser(qStr,req);
+ parser.setIsFilter(isFilter);
+ Query q = parser.getQuery();
+ if (rewrite) {
+ // TODO: do rewrite
+ }
+ ret += q.getClass().hashCode(); // use the query somehow
+ }
+ }
+
+ long end = System.nanoTime();
+
+ System.out.println((assertOn ? "WARNING, assertions enabled. " : "") + "ret=" + ret + " Parser QPS:" + ((long)numQueries * iter)*1000000000/(end-start));
+
+ req.close();
+ }
+
}
\ No newline at end of file