You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/02/26 17:33:46 UTC
svn commit: r1450264 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/java/org/apache/lucene/search/
lucene/core/src/test/org/apache/lucene/search/ solr/ solr/core/
solr/core/src...
Author: rmuir
Date: Tue Feb 26 16:33:45 2013
New Revision: 1450264
URL: http://svn.apache.org/r1450264
Log:
SOLR-4490: multi-valued dv support
Added:
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRangeFilter.java
- copied unchanged from r1450239, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRangeFilter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRewriteMethod.java
- copied unchanged from r1450239, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRewriteMethod.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRangeFilter.java
- copied unchanged from r1450239, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRangeFilter.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRewriteMethod.java
- copied unchanged from r1450239, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestDocTermOrdsRewriteMethod.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
- copied unchanged from r1450239, lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/DocValuesFacets.java
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml
- copied unchanged from r1450239, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMulti.xml
- copied unchanged from r1450239, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValuesMulti.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java
- copied, changed from r1450239, lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java
- copied, changed from r1450239, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TrieField.java
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java?rev=1450264&r1=1450263&r2=1450264&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java Tue Feb 26 16:33:45 2013
@@ -362,12 +362,19 @@ public class MultiDocValues {
}
}
- /** implements SortedDocValues over n subs, using an OrdinalMap */
- static class MultiSortedDocValues extends SortedDocValues {
- final int docStarts[];
- final SortedDocValues values[];
- final OrdinalMap mapping;
+ /**
+ * Implements SortedDocValues over n subs, using an OrdinalMap
+ * @lucene.internal
+ */
+ public static class MultiSortedDocValues extends SortedDocValues {
+ /** docbase for each leaf: parallel with {@link #values} */
+ public final int docStarts[];
+ /** leaf values */
+ public final SortedDocValues values[];
+ /** ordinal map mapping ords from <code>values</code> to global ord space */
+ public final OrdinalMap mapping;
+ /** Creates a new MultiSortedDocValues over <code>values</code> */
MultiSortedDocValues(SortedDocValues values[], int docStarts[], OrdinalMap mapping) throws IOException {
assert values.length == mapping.ordDeltas.length;
assert docStarts.length == values.length + 1;
@@ -396,13 +403,20 @@ public class MultiDocValues {
}
}
- /** implements MultiSortedDocValues over n subs, using an OrdinalMap */
- static class MultiSortedSetDocValues extends SortedSetDocValues {
- final int docStarts[];
- final SortedSetDocValues values[];
- final OrdinalMap mapping;
+ /**
+ * Implements MultiSortedSetDocValues over n subs, using an OrdinalMap
+ * @lucene.internal
+ */
+ public static class MultiSortedSetDocValues extends SortedSetDocValues {
+ /** docbase for each leaf: parallel with {@link #values} */
+ public final int docStarts[];
+ /** leaf values */
+ public final SortedSetDocValues values[];
+ /** ordinal map mapping ords from <code>values</code> to global ord space */
+ public final OrdinalMap mapping;
int currentSubIndex;
+ /** Creates a new MultiSortedSetDocValues over <code>values</code> */
MultiSortedSetDocValues(SortedSetDocValues values[], int docStarts[], OrdinalMap mapping) throws IOException {
assert values.length == mapping.ordDeltas.length;
assert docStarts.length == values.length + 1;
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1450264&r1=1450263&r2=1450264&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Tue Feb 26 16:33:45 2013
@@ -53,7 +53,7 @@ New Features
under the covers -- allowing many HTTP connection related properties to be
controlled via 'standard' java system properties. (hossman)
-* SOLR-3855: Doc values support. (Adrien Grand)
+* SOLR-3855, SOLR-4490: Doc values support. (Adrien Grand, Robert Muir)
* SOLR-4417: Reopen the IndexWriter on SolrCore reload. (Mark Miller)
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1450264&r1=1450263&r2=1450264&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Tue Feb 26 16:33:45 2013
@@ -389,6 +389,11 @@ public class SimpleFacets {
// only fc knows how to deal with multi-token fields
method = FacetMethod.FC;
}
+
+ if (method == FacetMethod.ENUM && sf.hasDocValues()) {
+ // only fc can handle docvalues types
+ method = FacetMethod.FC;
+ }
if (params.getFieldBool(field, GroupParams.GROUP_FACET, false)) {
counts = getGroupedCounts(searcher, docs, field, multiToken, offset,limit, mincount, missing, sort, prefix);
@@ -415,7 +420,9 @@ public class SimpleFacets {
}
break;
case FC:
- if (multiToken || TrieField.getMainValuePrefix(ft) != null) {
+ if (sf.hasDocValues()) {
+ counts = DocValuesFacets.getCounts(searcher, docs, field, offset,limit, mincount, missing, sort, prefix);
+ } else if (multiToken || TrieField.getMainValuePrefix(ft) != null) {
UnInvertedField uif = UnInvertedField.getUnInvertedField(field, searcher);
counts = uif.getCounts(searcher, docs, offset, limit, mincount,missing,sort,prefix);
} else {
@@ -536,9 +543,9 @@ public class SimpleFacets {
*/
public static int getFieldMissingCount(SolrIndexSearcher searcher, DocSet docs, String fieldName)
throws IOException {
-
+ SchemaField sf = searcher.getSchema().getField(fieldName);
DocSet hasVal = searcher.getDocSet
- (new TermRangeQuery(fieldName, null, null, false, false));
+ (sf.getType().getRangeQuery(null, sf, null, null, false, false));
return docs.andNotSize(hasVal);
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1450264&r1=1450263&r2=1450264&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/FieldType.java Tue Feb 26 16:33:45 2013
@@ -34,9 +34,10 @@ import org.apache.lucene.index.Indexable
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocTermOrdsRangeFilter;
+import org.apache.lucene.search.DocTermOrdsRewriteMethod;
import org.apache.lucene.search.FieldCacheRangeFilter;
import org.apache.lucene.search.FieldCacheRewriteMethod;
-import org.apache.lucene.search.FieldCacheTermsFilter;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
@@ -595,13 +596,21 @@ public abstract class FieldType extends
*
*/
public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
- // constant score mode is now enabled per default
+ // TODO: change these all to use readableToIndexed/bytes instead (e.g. for unicode collation)
if (field.hasDocValues() && !field.indexed()) {
- return new ConstantScoreQuery(FieldCacheRangeFilter.newStringRange(
+ if (field.multiValued()) {
+ return new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange(
+ field.getName(),
+ part1 == null ? null : new BytesRef(toInternal(part1)),
+ part2 == null ? null : new BytesRef(toInternal(part2)),
+ minInclusive, maxInclusive));
+ } else {
+ return new ConstantScoreQuery(FieldCacheRangeFilter.newStringRange(
field.getName(),
part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2),
minInclusive, maxInclusive));
+ }
} else {
MultiTermQuery rangeQuery = TermRangeQuery.newStringRange(
field.getName(),
@@ -626,7 +635,7 @@ public abstract class FieldType extends
readableToIndexed(externalVal, br);
if (field.hasDocValues() && !field.indexed()) {
// match-only
- return new ConstantScoreQuery(new FieldCacheTermsFilter(field.getName(), br));
+ return getRangeQuery(parser, field, externalVal, externalVal, true, true);
} else {
return new TermQuery(new Term(field.getName(), br));
}
@@ -640,7 +649,7 @@ public abstract class FieldType extends
*/
public MultiTermQuery.RewriteMethod getRewriteMethod(QParser parser, SchemaField field) {
if (!field.indexed() && field.hasDocValues()) {
- return new FieldCacheRewriteMethod();
+ return field.multiValued() ? new DocTermOrdsRewriteMethod() : new FieldCacheRewriteMethod();
} else {
return MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java?rev=1450264&r1=1450263&r2=1450264&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/StrField.java Tue Feb 26 16:33:45 2013
@@ -23,9 +23,9 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
@@ -46,8 +46,11 @@ public class StrField extends PrimitiveF
List<IndexableField> fields = new ArrayList<IndexableField>();
fields.add(createField(field, value, boost));
final BytesRef bytes = new BytesRef(value.toString());
- final Field docValuesField = new SortedDocValuesField(field.getName(), bytes);
- fields.add(docValuesField);
+ if (field.multiValued()) {
+ fields.add(new SortedSetDocValuesField(field.getName(), bytes));
+ } else {
+ fields.add(new SortedDocValuesField(field.getName(), bytes));
+ }
return fields;
} else {
return Collections.singletonList(createField(field, value, boost));
@@ -77,9 +80,8 @@ public class StrField extends PrimitiveF
@Override
public void checkSchemaField(SchemaField field) {
- // change me when multi-valued doc values are supported
- if (field.hasDocValues() && !(field.isRequired() || field.getDefaultValue() != null)) {
- throw new IllegalStateException("Field " + this + " has doc values enabled, but has no default value and is not required");
+ if (field.hasDocValues() && !field.multiValued() && !(field.isRequired() || field.getDefaultValue() != null)) {
+ throw new IllegalStateException("Field " + this + " has single-valued doc values enabled, but has no default value and is not required");
}
}
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TrieField.java?rev=1450264&r1=1450263&r2=1450264&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/schema/TrieField.java Tue Feb 26 16:33:45 2013
@@ -34,6 +34,7 @@ import org.apache.lucene.document.IntFie
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
@@ -263,6 +264,10 @@ public class TrieField extends Primitive
@Override
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
+ if (field.multiValued() && field.hasDocValues() && !field.indexed()) {
+ // for the multi-valued dv-case, the default rangeimpl over toInternal is correct
+ return super.getRangeQuery(parser, field, min, max, minInclusive, maxInclusive);
+ }
int ps = precisionStep;
Query query = null;
final boolean matchOnly = field.hasDocValues() && !field.indexed();
@@ -627,16 +632,24 @@ public class TrieField extends Primitive
List<IndexableField> fields = new ArrayList<IndexableField>();
final IndexableField field = createField(sf, value, boost);
fields.add(field);
- final long bits;
- if (field.numericValue() instanceof Integer || field.numericValue() instanceof Long) {
- bits = field.numericValue().longValue();
- } else if (field.numericValue() instanceof Float) {
- bits = Float.floatToIntBits(field.numericValue().floatValue());
+
+ if (sf.multiValued()) {
+ BytesRef bytes = new BytesRef();
+ readableToIndexed(value.toString(), bytes);
+ fields.add(new SortedSetDocValuesField(sf.getName(), bytes));
} else {
- assert field.numericValue() instanceof Double;
- bits = Double.doubleToLongBits(field.numericValue().doubleValue());
+ final long bits;
+ if (field.numericValue() instanceof Integer || field.numericValue() instanceof Long) {
+ bits = field.numericValue().longValue();
+ } else if (field.numericValue() instanceof Float) {
+ bits = Float.floatToIntBits(field.numericValue().floatValue());
+ } else {
+ assert field.numericValue() instanceof Double;
+ bits = Double.doubleToLongBits(field.numericValue().doubleValue());
+ }
+ fields.add(new NumericDocValuesField(sf.getName(), bits));
}
- fields.add(new NumericDocValuesField(sf.getName(), bits));
+
return fields;
} else {
return Collections.singletonList(createField(sf, value, boost));
@@ -683,8 +696,8 @@ public class TrieField extends Primitive
@Override
public void checkSchemaField(final SchemaField field) {
- if (field.hasDocValues() && !(field.isRequired() || field.getDefaultValue() != null)) {
- throw new IllegalStateException("Field " + this + " has doc values enabled, but has no default value and is not required");
+ if (field.hasDocValues() && !field.multiValued() && !(field.isRequired() || field.getDefaultValue() != null)) {
+ throw new IllegalStateException("Field " + this + " has single-valued doc values enabled, but has no default value and is not required");
}
}
}
Copied: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java (from r1450239, lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java?p2=lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java&p1=lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java&r1=1450239&r2=1450264&rev=1450264&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/TestRandomDVFaceting.java Tue Feb 26 16:33:45 2013
@@ -39,7 +39,7 @@ import org.junit.Test;
* to the indexed facet results as if it were just another faceting method.
*/
@Slow
-@SuppressCodecs({"Lucene40", "Lucene41"})
+@SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41"})
public class TestRandomDVFaceting extends SolrTestCaseJ4 {
@BeforeClass
Copied: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java (from r1450239, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java?p2=lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java&p1=lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java&r1=1450239&r2=1450264&rev=1450264&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/schema/DocValuesMultiTest.java Tue Feb 26 16:33:45 2013
@@ -30,7 +30,7 @@ import org.apache.solr.search.SolrIndexS
import org.apache.solr.util.RefCounted;
import org.junit.BeforeClass;
-@SuppressCodecs({"Lucene40", "Lucene41"})
+@SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41"})
public class DocValuesMultiTest extends SolrTestCaseJ4 {
@BeforeClass