You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/02/20 23:12:30 UTC
svn commit: r1448440 - in /lucene/dev/trunk:
lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/search/ solr/
solr/core/src/java/org/apache/solr/parser/
solr/core/src/java/org/apache/solr/schema/ solr/core/src/test/or...
Author: rmuir
Date: Wed Feb 20 22:12:29 2013
New Revision: 1448440
URL: http://svn.apache.org/r1448440
Log:
SOLR-4477: match-only query support for docvalues fields
Added:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java
- copied, changed from r1448436, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java
Removed:
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java
Copied: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java (from r1448436, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java?p2=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java&r1=1448436&r2=1448440&rev=1448440&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java Wed Feb 20 22:12:29 2013
@@ -33,11 +33,7 @@ import org.apache.lucene.util.OpenBitSet
/**
* Rewrites MultiTermQueries into a filter, using the FieldCache for term enumeration.
* <p>
- * WARNING: This is only appropriate for single-valued unanalyzed fields. Additionally, for
- * most queries this method is actually SLOWER than using the default CONSTANT_SCORE_AUTO
- * in MultiTermQuery. This method is only faster than other methods for certain queries,
- * such as ones that enumerate many terms.
- *
+ * This can be used to perform these queries against an unindexed docvalues field.
* @lucene.experimental
*/
public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod {
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1448440&r1=1448439&r2=1448440&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Wed Feb 20 22:12:29 2013
@@ -75,6 +75,9 @@ New Features
* SOLR-4417: Reopen the IndexWriter on SolrCore reload. (Mark Miller)
+* SOLR-4477: Add support for queries (match-only) against docvalues fields.
+ (Robert Muir)
+
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java?rev=1448440&r1=1448439&r2=1448440&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/parser/SolrQueryParserBase.java Wed Feb 20 22:12:29 2013
@@ -626,7 +626,8 @@ public abstract class SolrQueryParserBas
*/
protected Query newPrefixQuery(Term prefix){
PrefixQuery query = new PrefixQuery(prefix);
- query.setRewriteMethod(multiTermRewriteMethod);
+ SchemaField sf = schema.getField(prefix.field());
+ query.setRewriteMethod(sf.getType().getRewriteMethod(parser, sf));
return query;
}
@@ -637,7 +638,8 @@ public abstract class SolrQueryParserBas
*/
protected Query newRegexpQuery(Term regexp) {
RegexpQuery query = new RegexpQuery(regexp);
- query.setRewriteMethod(multiTermRewriteMethod);
+ SchemaField sf = schema.getField(regexp.field());
+ query.setRewriteMethod(sf.getType().getRewriteMethod(parser, sf));
return query;
}
@@ -671,7 +673,8 @@ public abstract class SolrQueryParserBas
*/
protected Query newWildcardQuery(Term t) {
WildcardQuery query = new WildcardQuery(t);
- query.setRewriteMethod(multiTermRewriteMethod);
+ SchemaField sf = schema.getField(t.field());
+ query.setRewriteMethod(sf.getType().getRewriteMethod(parser, sf));
return query;
}
@@ -934,7 +937,7 @@ public abstract class SolrQueryParserBas
if (sf != null) {
FieldType ft = sf.getType();
// delegate to type for everything except tokenized fields
- if (ft.isTokenized()) {
+ if (ft.isTokenized() && sf.indexed()) {
return newFieldQuery(analyzer, field, queryText, quoted || (ft instanceof TextField && ((TextField)ft).getAutoGeneratePhraseQueries()));
} else {
return sf.getType().getFieldQuery(parser, sf, queryText);
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1448440&r1=1448439&r2=1448440&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java Wed Feb 20 22:12:29 2013
@@ -34,6 +34,11 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.FieldCacheRangeFilter;
+import org.apache.lucene.search.FieldCacheRewriteMethod;
+import org.apache.lucene.search.FieldCacheTermsFilter;
+import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
@@ -592,11 +597,21 @@ public abstract class FieldType extends
*/
public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
// constant score mode is now enabled per default
- return TermRangeQuery.newStringRange(
+ if (field.hasDocValues() && !field.indexed()) {
+ return new ConstantScoreQuery(FieldCacheRangeFilter.newStringRange(
+ field.getName(),
+ part1 == null ? null : toInternal(part1),
+ part2 == null ? null : toInternal(part2),
+ minInclusive, maxInclusive));
+ } else {
+ MultiTermQuery rangeQuery = TermRangeQuery.newStringRange(
field.getName(),
part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2),
minInclusive, maxInclusive);
+ rangeQuery.setRewriteMethod(getRewriteMethod(parser, field));
+ return rangeQuery;
+ }
}
/**
@@ -610,7 +625,26 @@ public abstract class FieldType extends
public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) {
BytesRef br = new BytesRef();
readableToIndexed(externalVal, br);
- return new TermQuery(new Term(field.getName(), br));
+ if (field.hasDocValues() && !field.indexed()) {
+ // match-only
+ return new ConstantScoreQuery(new FieldCacheTermsFilter(field.getName(), br));
+ } else {
+ return new TermQuery(new Term(field.getName(), br));
+ }
+ }
+
+ /**
+ * Expert: Returns the rewrite method for multiterm queries such as wildcards.
+ * @param parser The {@link org.apache.solr.search.QParser} calling the method
+ * @param field The {@link org.apache.solr.schema.SchemaField} of the field to search
+ * @return A suitable rewrite method for rewriting multi-term queries to primitive queries.
+ */
+ public MultiTermQuery.RewriteMethod getRewriteMethod(QParser parser, SchemaField field) {
+ if (!field.indexed() && field.hasDocValues()) {
+ return new FieldCacheRewriteMethod();
+ } else {
+ return MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT;
+ }
}
/**
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java?rev=1448440&r1=1448439&r2=1448440&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java Wed Feb 20 22:12:29 2013
@@ -39,7 +39,9 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
+import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
@@ -263,36 +265,72 @@ public class TrieField extends Primitive
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
int ps = precisionStep;
Query query = null;
+ final boolean matchOnly = field.hasDocValues() && !field.indexed();
switch (type) {
case INTEGER:
- query = NumericRangeQuery.newIntRange(field.getName(), ps,
+ if (matchOnly) {
+ query = new ConstantScoreQuery(FieldCacheRangeFilter.newIntRange(field.getName(),
+ min == null ? null : Integer.parseInt(min),
+ max == null ? null : Integer.parseInt(max),
+ minInclusive, maxInclusive));
+ } else {
+ query = NumericRangeQuery.newIntRange(field.getName(), ps,
min == null ? null : Integer.parseInt(min),
max == null ? null : Integer.parseInt(max),
minInclusive, maxInclusive);
+ }
break;
case FLOAT:
- query = NumericRangeQuery.newFloatRange(field.getName(), ps,
+ if (matchOnly) {
+ query = new ConstantScoreQuery(FieldCacheRangeFilter.newFloatRange(field.getName(),
+ min == null ? null : Float.parseFloat(min),
+ max == null ? null : Float.parseFloat(max),
+ minInclusive, maxInclusive));
+ } else {
+ query = NumericRangeQuery.newFloatRange(field.getName(), ps,
min == null ? null : Float.parseFloat(min),
max == null ? null : Float.parseFloat(max),
minInclusive, maxInclusive);
+ }
break;
case LONG:
- query = NumericRangeQuery.newLongRange(field.getName(), ps,
+ if (matchOnly) {
+ query = new ConstantScoreQuery(FieldCacheRangeFilter.newLongRange(field.getName(),
+ min == null ? null : Long.parseLong(min),
+ max == null ? null : Long.parseLong(max),
+ minInclusive, maxInclusive));
+ } else {
+ query = NumericRangeQuery.newLongRange(field.getName(), ps,
min == null ? null : Long.parseLong(min),
max == null ? null : Long.parseLong(max),
minInclusive, maxInclusive);
+ }
break;
case DOUBLE:
- query = NumericRangeQuery.newDoubleRange(field.getName(), ps,
+ if (matchOnly) {
+ query = new ConstantScoreQuery(FieldCacheRangeFilter.newDoubleRange(field.getName(),
+ min == null ? null : Double.parseDouble(min),
+ max == null ? null : Double.parseDouble(max),
+ minInclusive, maxInclusive));
+ } else {
+ query = NumericRangeQuery.newDoubleRange(field.getName(), ps,
min == null ? null : Double.parseDouble(min),
max == null ? null : Double.parseDouble(max),
minInclusive, maxInclusive);
+ }
break;
case DATE:
- query = NumericRangeQuery.newLongRange(field.getName(), ps,
+ if (matchOnly) {
+ query = new ConstantScoreQuery(FieldCacheRangeFilter.newLongRange(field.getName(),
+ min == null ? null : dateField.parseMath(null, min).getTime(),
+ max == null ? null : dateField.parseMath(null, max).getTime(),
+ minInclusive, maxInclusive));
+ } else {
+ query = NumericRangeQuery.newLongRange(field.getName(), ps,
min == null ? null : dateField.parseMath(null, min).getTime(),
max == null ? null : dateField.parseMath(null, max).getTime(),
minInclusive, maxInclusive);
+ }
break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field");
@@ -300,6 +338,16 @@ public class TrieField extends Primitive
return query;
}
+
+ @Override
+ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) {
+ if (!field.indexed() && field.hasDocValues()) {
+ // currently implemented as singleton range
+ return getRangeQuery(parser, field, externalVal, externalVal, true, true);
+ } else {
+ return super.getFieldQuery(parser, field, externalVal);
+ }
+ }
@Deprecated
static int toInt(byte[] arr, int offset) {
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java?rev=1448440&r1=1448439&r2=1448440&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java Wed Feb 20 22:12:29 2013
@@ -226,5 +226,92 @@ public class DocValuesTest extends SolrT
"//lst[@name='datedv']/lst[@name='1904-12-31T23:59:59.999Z']/long[@name='count'][.='8']",
"//lst[@name='datedv']/lst[@name='1905-12-31T23:59:59.999Z']/long[@name='count'][.='8']");
}
-
+
+ /** Tests the ability to do basic queries (without scoring, just match-only) on
+ * docvalues fields that are not inverted (indexed "forward" only)
+ */
+ public void testDocValuesMatch() throws Exception {
+ assertU(adoc("id", "1", "floatdv", "2", "intdv", "3", "doubledv", "4", "longdv", "5", "datedv", "1995-12-31T23:59:59.999Z", "stringdv", "b"));
+ assertU(adoc("id", "2", "floatdv", "5", "intdv", "4", "doubledv", "3", "longdv", "2", "datedv", "1997-12-31T23:59:59.999Z", "stringdv", "a"));
+ assertU(adoc("id", "3", "floatdv", "3", "intdv", "1", "doubledv", "2", "longdv", "1", "datedv", "1996-12-31T23:59:59.999Z", "stringdv", "c"));
+ assertU(adoc("id", "4", "floatdv", "3", "intdv", "1", "doubledv", "2", "longdv", "1", "datedv", "1996-12-31T23:59:59.999Z", "stringdv", "car"));
+ assertU(commit());
+
+ // string: termquery
+ assertQ(req("q", "stringdv:car", "sort", "id asc"),
+ "//*[@numFound='1']",
+ "//result/doc[1]/str[@name='id'][.=4]"
+ );
+
+ // string: range query
+ assertQ(req("q", "stringdv:[b TO d]", "sort", "id asc"),
+ "//*[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.=1]",
+ "//result/doc[2]/str[@name='id'][.=3]",
+ "//result/doc[3]/str[@name='id'][.=4]"
+ );
+
+ // string: prefix query
+ assertQ(req("q", "stringdv:c*", "sort", "id asc"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.=3]",
+ "//result/doc[2]/str[@name='id'][.=4]"
+ );
+
+ // string: wildcard query
+ assertQ(req("q", "stringdv:c?r", "sort", "id asc"),
+ "//*[@numFound='1']",
+ "//result/doc[1]/str[@name='id'][.=4]"
+ );
+
+ // string: regexp query
+ assertQ(req("q", "stringdv:/c[a-b]r/", "sort", "id asc"),
+ "//*[@numFound='1']",
+ "//result/doc[1]/str[@name='id'][.=4]"
+ );
+
+ // float: termquery
+ assertQ(req("q", "floatdv:3", "sort", "id asc"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.=3]",
+ "//result/doc[2]/str[@name='id'][.=4]"
+ );
+
+ // float: rangequery
+ assertQ(req("q", "floatdv:[2 TO 3]", "sort", "id asc"),
+ "//*[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.=1]",
+ "//result/doc[2]/str[@name='id'][.=3]",
+ "//result/doc[3]/str[@name='id'][.=4]"
+ );
+
+ // int: termquery
+ assertQ(req("q", "intdv:1", "sort", "id asc"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.=3]",
+ "//result/doc[2]/str[@name='id'][.=4]"
+ );
+
+ // int: rangequery
+ assertQ(req("q", "intdv:[3 TO 4]", "sort", "id asc"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.=1]",
+ "//result/doc[2]/str[@name='id'][.=2]"
+ );
+
+ // long: termquery
+ assertQ(req("q", "longdv:1", "sort", "id asc"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.=3]",
+ "//result/doc[2]/str[@name='id'][.=4]"
+ );
+
+ // long: rangequery
+ assertQ(req("q", "longdv:[1 TO 2]", "sort", "id asc"),
+ "//*[@numFound='3']",
+ "//result/doc[1]/str[@name='id'][.=2]",
+ "//result/doc[2]/str[@name='id'][.=3]",
+ "//result/doc[3]/str[@name='id'][.=4]"
+ );
+ }
}