You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/02/16 19:50:21 UTC
svn commit: r1446922 [1/2] - in /lucene/dev/trunk:
lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/
solr/ solr/core/src/java/org/apache/solr/core/
solr/core/src/java/org/apache/solr/handler/component/
solr/core/src/java/org/apach...
Author: jpountz
Date: Sat Feb 16 18:50:20 2013
New Revision: 1446922
URL: http://svn.apache.org/r1446922
Log:
SOLR-3855: Doc values support.
Added:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java (with props)
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/bad-schema-docValues-not-required-no-default.xml (with props)
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/bad-schema-unsupported-docValues.xml (with props)
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml (with props)
lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/DocValuesTest.java (with props)
Modified:
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldProperties.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/LatLonType.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PointType.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieDateField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/UUIDField.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DocumentBuilder.java
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema.xml
lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema_codec.xml
lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/PolyFieldTest.java
lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java Sat Feb 16 18:50:20 2013
@@ -85,7 +85,7 @@ public class IntFieldSource extends Fiel
@Override
public String strVal(int doc) {
- return Float.toString(arr.get(doc));
+ return Integer.toString(arr.get(doc));
}
@Override
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java Sat Feb 16 18:50:20 2013
@@ -64,6 +64,10 @@ public class LongFieldSource extends Fie
return val;
}
+ public String longToString(long val) {
+ return longToObject(val).toString();
+ }
+
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
final FieldCache.Longs arr = cache.getLongs(readerContext.reader(), field, parser, true);
@@ -86,6 +90,11 @@ public class LongFieldSource extends Fie
}
@Override
+ public String strVal(int doc) {
+ return valid.get(doc) ? longToString(arr.get(doc)) : null;
+ }
+
+ @Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
long lower,upper;
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Sat Feb 16 18:50:20 2013
@@ -77,6 +77,8 @@ New Features
under the covers -- allowing many HTTP connection related properties to be
controlled via 'standard' java system properties. (hossman)
+* SOLR-3855: Doc values support. (Adrien Grand)
+
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecFactory.java Sat Feb 16 18:50:20 2013
@@ -1,6 +1,7 @@
package org.apache.solr.core;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene42.Lucene42Codec;
import org.apache.solr.schema.IndexSchema;
@@ -55,7 +56,18 @@ public class SchemaCodecFactory extends
}
return super.getPostingsFormatForField(field);
}
- // TODO: when dv support is added to solr, add it here too
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ final SchemaField fieldOrNull = schema.getFieldOrNull(field);
+ if (fieldOrNull == null) {
+ throw new IllegalArgumentException("no such field " + field);
+ }
+ String docValuesFormatName = fieldOrNull.getType().getDocValuesFormat();
+ if (docValuesFormatName != null) {
+ return DocValuesFormat.forName(docValuesFormatName);
+ }
+ return super.getDocValuesFormatForField(field);
+ }
};
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java Sat Feb 16 18:50:20 2013
@@ -834,6 +834,11 @@ public final class SolrCore implements S
log.error(msg);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
}
+ if (null != ft.getDocValuesFormat()) {
+ String msg = "FieldType '" + ft.getTypeName() + "' is configured with a docValues format, but the codec does not support it: " + factory.getClass();
+ log.error(msg);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
+ }
}
}
return factory.getCodec();
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FieldFacetStats.java Sat Feb 16 18:50:20 2013
@@ -16,16 +16,22 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
-import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.SolrIndexSearcher;
/**
@@ -40,101 +46,76 @@ import org.apache.solr.schema.SchemaFiel
public class FieldFacetStats {
public final String name;
- final SortedDocValues si;
final SchemaField facet_sf;
final SchemaField field_sf;
- final int startTermIndex;
- final int endTermIndex;
- final int nTerms;
+ public final Map<String, StatsValues> facetStatsValues;
- final int numStatsTerms;
+ List<HashMap<String, Integer>> facetStatsTerms;
- public final Map<String, StatsValues> facetStatsValues;
+ final AtomicReader topLevelReader;
+ AtomicReaderContext leave;
+ final ValueSource valueSource;
+ AtomicReaderContext context;
+ FunctionValues values;
- final List<HashMap<String, Integer>> facetStatsTerms;
+ SortedDocValues topLevelSortedValues = null;
private final BytesRef tempBR = new BytesRef();
- public FieldFacetStats(String name, SortedDocValues si, SchemaField field_sf, SchemaField facet_sf, int numStatsTerms) {
+ public FieldFacetStats(SolrIndexSearcher searcher, String name, SchemaField field_sf, SchemaField facet_sf) {
this.name = name;
- this.si = si;
this.field_sf = field_sf;
this.facet_sf = facet_sf;
- this.numStatsTerms = numStatsTerms;
- startTermIndex = 0;
- endTermIndex = si.getValueCount();
- nTerms = endTermIndex - startTermIndex;
+ topLevelReader = searcher.getAtomicReader();
+ valueSource = facet_sf.getType().getValueSource(facet_sf, null);
facetStatsValues = new HashMap<String, StatsValues>();
-
- // for mv stats field, we'll want to keep track of terms
facetStatsTerms = new ArrayList<HashMap<String, Integer>>();
- if (numStatsTerms == 0) return;
- int i = 0;
- for (; i < numStatsTerms; i++) {
- facetStatsTerms.add(new HashMap<String, Integer>());
- }
}
- BytesRef getTermText(int docID, BytesRef ret) {
- final int ord = si.getOrd(docID);
- if (ord == -1) {
- return null;
- } else {
- si.lookupOrd(ord, ret);
- return ret;
+ private StatsValues getStatsValues(String key) throws IOException {
+ StatsValues stats = facetStatsValues.get(key);
+ if (stats == null) {
+ stats = StatsValuesFactory.createStatsValues(field_sf);
+ facetStatsValues.put(key, stats);
+ stats.setNextReader(context);
}
+ return stats;
}
- public boolean facet(int docID, BytesRef v) {
- int term = si.getOrd(docID);
- int arrIdx = term - startTermIndex;
- if (arrIdx >= 0 && arrIdx < nTerms) {
-
- final BytesRef br;
- if (term == -1) {
- br = null;
- } else {
- br = tempBR;
- si.lookupOrd(term, tempBR);
- }
- String key = (br == null)?null:facet_sf.getType().indexedToReadable(br.utf8ToString());
- StatsValues stats = facetStatsValues.get(key);
- if (stats == null) {
- stats = StatsValuesFactory.createStatsValues(field_sf);
- facetStatsValues.put(key, stats);
- }
-
- if (v != null && v.length>0) {
- stats.accumulate(v);
- } else {
- stats.missing();
- return false;
- }
- return true;
- }
- return false;
+ // docID is relative to the context
+ public void facet(int docID) throws IOException {
+ final String key = values.exists(docID)
+ ? values.strVal(docID)
+ : null;
+ final StatsValues stats = getStatsValues(key);
+ stats.accumulate(docID);
}
-
// Function to keep track of facet counts for term number.
// Currently only used by UnInvertedField stats
- public boolean facetTermNum(int docID, int statsTermNum) {
-
- int term = si.getOrd(docID);
- int arrIdx = term - startTermIndex;
- if (arrIdx >= 0 && arrIdx < nTerms) {
+ public boolean facetTermNum(int docID, int statsTermNum) throws IOException {
+ if (topLevelSortedValues == null) {
+ topLevelSortedValues = FieldCache.DEFAULT.getTermsIndex(topLevelReader, name);
+ }
+
+ int term = topLevelSortedValues.getOrd(docID);
+ int arrIdx = term;
+ if (arrIdx >= 0 && arrIdx < topLevelSortedValues.getValueCount()) {
final BytesRef br;
if (term == -1) {
br = null;
} else {
br = tempBR;
- si.lookupOrd(term, tempBR);
+ topLevelSortedValues.lookupOrd(term, tempBR);
}
String key = br == null ? null : br.utf8ToString();
- HashMap<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
+ while (facetStatsTerms.size() <= statsTermNum) {
+ facetStatsTerms.add(new HashMap<String, Integer>());
+ }
+ final Map<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
Integer statsTermCount = statsTermCounts.get(key);
if (statsTermCount == null) {
statsTermCounts.put(key, 1);
@@ -148,8 +129,11 @@ public class FieldFacetStats {
//function to accumulate counts for statsTermNum to specified value
- public boolean accumulateTermNum(int statsTermNum, BytesRef value) {
+ public boolean accumulateTermNum(int statsTermNum, BytesRef value) throws IOException {
if (value == null) return false;
+ while (facetStatsTerms.size() <= statsTermNum) {
+ facetStatsTerms.add(new HashMap<String, Integer>());
+ }
for (Map.Entry<String, Integer> stringIntegerEntry : facetStatsTerms.get(statsTermNum).entrySet()) {
Map.Entry pairs = (Map.Entry) stringIntegerEntry;
String key = (String) pairs.getKey();
@@ -166,6 +150,14 @@ public class FieldFacetStats {
return true;
}
+ public void setNextReader(AtomicReaderContext ctx) throws IOException {
+ this.context = ctx;
+ values = valueSource.getValues(Collections.emptyMap(), ctx);
+ for (StatsValues stats : facetStatsValues.values()) {
+ stats.setNextReader(ctx);
+ }
+ }
+
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java Sat Feb 16 18:50:20 2013
@@ -20,12 +20,11 @@ package org.apache.solr.handler.componen
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
@@ -43,14 +42,12 @@ import org.apache.solr.search.SolrIndexS
/**
* Stats component calculates simple statistics on numeric field values
- *
- *
* @since solr 1.4
*/
public class StatsComponent extends SearchComponent {
public static final String COMPONENT_NAME = "stats";
-
+
@Override
public void prepare(ResponseBuilder rb) throws IOException {
if (rb.req.getParams().getBool(StatsParams.STATS,false)) {
@@ -236,25 +233,13 @@ class SimpleStats {
}
return res;
}
-
- // why does this use a top-level field cache?
- public NamedList<?> getFieldCacheStats(String fieldName, String[] facet ) {
- SchemaField sf = searcher.getSchema().getField(fieldName);
-
- SortedDocValues si;
- try {
- si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), fieldName);
- }
- catch (IOException e) {
- throw new RuntimeException( "failed to open field cache for: "+fieldName, e );
- }
- StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
- final int nTerms = si.getValueCount();
- if ( nTerms <= 0 || docs.size() <= 0 ) return allstats.getStatsValues();
- // don't worry about faceting if no documents match...
+ public NamedList<?> getFieldCacheStats(String fieldName, String[] facet) throws IOException {
+ final SchemaField sf = searcher.getSchema().getField(fieldName);
+
+ final StatsValues allstats = StatsValuesFactory.createStatsValues(sf);
+
List<FieldFacetStats> facetStats = new ArrayList<FieldFacetStats>();
- SortedDocValues facetTermsIndex;
for( String facetField : facet ) {
SchemaField fsf = searcher.getSchema().getField(facetField);
@@ -262,40 +247,32 @@ class SimpleStats {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Stats can only facet on single-valued fields, not: " + facetField );
}
-
- try {
- facetTermsIndex = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), facetField);
- }
- catch (IOException e) {
- throw new RuntimeException( "failed to open field cache for: "
- + facetField, e );
- }
- facetStats.add(new FieldFacetStats(facetField, facetTermsIndex, sf, fsf, nTerms));
+
+ facetStats.add(new FieldFacetStats(searcher, facetField, sf, fsf));
}
-
- final BytesRef tempBR = new BytesRef();
- DocIterator iter = docs.iterator();
- while (iter.hasNext()) {
- int docID = iter.nextDoc();
- int docOrd = si.getOrd(docID);
- BytesRef raw;
- if (docOrd == -1) {
- allstats.missing();
- tempBR.length = 0;
- raw = tempBR;
- } else {
- raw = tempBR;
- si.lookupOrd(docOrd, tempBR);
- if( tempBR.length > 0 ) {
- allstats.accumulate(tempBR);
- } else {
- allstats.missing();
+
+ final Iterator<AtomicReaderContext> ctxIt = searcher.getIndexReader().leaves().iterator();
+ AtomicReaderContext ctx = null;
+ for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
+ final int doc = docsIt.nextDoc();
+ if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
+ // advance
+ do {
+ ctx = ctxIt.next();
+ } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc());
+ assert doc >= ctx.docBase;
+
+ // propagate the context among accumulators.
+ allstats.setNextReader(ctx);
+ for (FieldFacetStats f : facetStats) {
+ f.setNextReader(ctx);
}
}
- // now update the facets
+ // accumulate
+ allstats.accumulate(doc - ctx.docBase);
for (FieldFacetStats f : facetStats) {
- f.facet(docID, raw);
+ f.facet(doc - ctx.docBase);
}
}
@@ -305,5 +282,4 @@ class SimpleStats {
return allstats.getStatsValues();
}
-
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValues.java Sat Feb 16 18:50:20 2013
@@ -19,14 +19,19 @@
package org.apache.solr.handler.component;
+import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.schema.FieldType;
/**
* StatsValue defines the interface for the collection of statistical values about fields and facets.
*/
+// TODO: should implement Collector?
public interface StatsValues {
/**
@@ -36,12 +41,9 @@ public interface StatsValues {
*/
void accumulate(NamedList stv);
- /**
- * Accumulate the values based on the given value
- *
- * @param value Value to use to accumulate the current values
- */
- void accumulate(BytesRef value);
+ /** Accumulate the value associated with <code>docID</code>.
+ * @see #setNextReader(AtomicReaderContext) */
+ void accumulate(int docID);
/**
* Accumulate the values based on the given value
@@ -77,4 +79,7 @@ public interface StatsValues {
* @return NamedList representation of the current values
*/
NamedList<?> getStatsValues();
+
+ /** Set the context for {@link #accumulate(int)}. */
+ void setNextReader(AtomicReaderContext ctx) throws IOException;
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java Sat Feb 16 18:50:20 2013
@@ -17,10 +17,15 @@
package org.apache.solr.handler.component;
+import java.io.IOException;
+import java.util.Collections;
import java.util.Date;
import java.util.Map;
import java.util.HashMap;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
@@ -39,6 +44,7 @@ public class StatsValuesFactory {
* @return Instance of StatsValues that will create statistics from values from a field of the given type
*/
public static StatsValues createStatsValues(SchemaField sf) {
+ // TODO: allow for custom field types
FieldType fieldType = sf.getType();
if (DoubleField.class.isInstance(fieldType) ||
IntField.class.isInstance(fieldType) ||
@@ -77,6 +83,8 @@ abstract class AbstractStatsValues<T> im
protected T min;
protected long missing;
protected long count;
+ private ValueSource valueSource;
+ protected FunctionValues values;
// facetField facetValue
protected Map<String, Map<String, StatsValues>> facets = new HashMap<String, Map<String, StatsValues>>();
@@ -121,29 +129,22 @@ abstract class AbstractStatsValues<T> im
}
}
}
-
+
/**
* {@inheritDoc}
*/
@Override
- public void accumulate(BytesRef value) {
- count++;
+ public void accumulate(BytesRef value, int count) {
T typedValue = (T)ft.toObject(sf, value);
- updateMinMax(typedValue, typedValue);
- updateTypeSpecificStats(typedValue);
+ accumulate(typedValue, count);
}
- /**
- * {@inheritDoc}
- */
- @Override
- public void accumulate(BytesRef value, int count) {
+ public void accumulate(T value, int count) {
this.count += count;
- T typedValue = (T)ft.toObject(sf, value);
- updateMinMax(typedValue, typedValue);
- updateTypeSpecificStats(typedValue, count);
+ updateMinMax(value, value);
+ updateTypeSpecificStats(value, count);
}
-
+
/**
* {@inheritDoc}
*/
@@ -194,6 +195,13 @@ abstract class AbstractStatsValues<T> im
return res;
}
+ public void setNextReader(AtomicReaderContext ctx) throws IOException {
+ if (valueSource == null) {
+ valueSource = ft.getValueSource(sf, null);
+ }
+ values = valueSource.getValues(Collections.emptyMap(), ctx);
+ }
+
/**
* Updates the minimum and maximum statistics based on the given values
*
@@ -206,13 +214,6 @@ abstract class AbstractStatsValues<T> im
* Updates the type specific statistics based on the given value
*
* @param value Value the statistics should be updated against
- */
- protected abstract void updateTypeSpecificStats(T value);
-
- /**
- * Updates the type specific statistics based on the given value
- *
- * @param value Value the statistics should be updated against
* @param count Number of times the value is being accumulated
*/
protected abstract void updateTypeSpecificStats(T value, int count);
@@ -246,23 +247,22 @@ class NumericStatsValues extends Abstrac
max = Double.NEGATIVE_INFINITY;
}
- /**
- * {@inheritDoc}
- */
@Override
- public void updateTypeSpecificStats(NamedList stv) {
- sum += ((Number)stv.get("sum")).doubleValue();
- sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
+ public void accumulate(int docID) {
+ if (values.exists(docID)) {
+ accumulate((Number) values.objectVal(docID), 1);
+ } else {
+ missing();
+ }
}
/**
* {@inheritDoc}
*/
@Override
- public void updateTypeSpecificStats(Number v) {
- double value = v.doubleValue();
- sumOfSquares += (value * value); // for std deviation
- sum += value;
+ public void updateTypeSpecificStats(NamedList stv) {
+ sum += ((Number)stv.get("sum")).doubleValue();
+ sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
}
/**
@@ -323,23 +323,22 @@ class DateStatsValues extends AbstractSt
super(sf);
}
- /**
- * {@inheritDoc}
- */
@Override
- protected void updateTypeSpecificStats(NamedList stv) {
- sum += ((Date) stv.get("sum")).getTime();
- sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
+ public void accumulate(int docID) {
+ if (values.exists(docID)) {
+ accumulate((Date) values.objectVal(docID), 1);
+ } else {
+ missing();
+ }
}
/**
* {@inheritDoc}
*/
@Override
- public void updateTypeSpecificStats(Date v) {
- long value = v.getTime();
- sumOfSquares += (value * value); // for std deviation
- sum += value;
+ protected void updateTypeSpecificStats(NamedList stv) {
+ sum += ((Date) stv.get("sum")).getTime();
+ sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
}
/**
@@ -407,19 +406,20 @@ class StringStatsValues extends Abstract
super(sf);
}
- /**
- * {@inheritDoc}
- */
@Override
- protected void updateTypeSpecificStats(NamedList stv) {
- // No type specific stats
+ public void accumulate(int docID) {
+ if (values.exists(docID)) {
+ accumulate(values.strVal(docID), 1);
+ } else {
+ missing();
+ }
}
/**
* {@inheritDoc}
*/
@Override
- protected void updateTypeSpecificStats(String value) {
+ protected void updateTypeSpecificStats(NamedList stv) {
// No type specific stats
}
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java?rev=1446922&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/NumericFacets.java Sat Feb 16 18:50:20 2013
@@ -0,0 +1,328 @@
+package org.apache.solr.request;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lucene.document.FieldType.NumericType;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.ReaderUtil;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.queries.function.FunctionValues;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.solr.common.params.FacetParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.SolrIndexSearcher;
+
+/** Utility class to compute facets on numeric fields. */
+final class NumericFacets {
+
+ NumericFacets() {}
+
+ static class HashTable {
+
+ static final float LOAD_FACTOR = 0.7f;
+
+ long[] bits; // bits identifying a value
+ int[] counts;
+ int[] docIDs;
+ int mask;
+ int size;
+ int threshold;
+
+ HashTable() {
+ final int capacity = 64; // must be a power of 2
+ bits = new long[capacity];
+ counts = new int[capacity];
+ docIDs = new int[capacity];
+ mask = capacity - 1;
+ size = 0;
+ threshold = (int) (capacity * LOAD_FACTOR);
+ }
+
+ private int hash(long v) {
+ int h = (int) (v ^ (v >>> 32));
+ h = (31 * h) & mask; // * 31 to try to use the whole table, even if values are dense
+ return h;
+ }
+
+ void add(int docID, long value, int count) {
+ if (size >= threshold) {
+ rehash();
+ }
+ final int h = hash(value);
+ for (int slot = h; ; slot = (slot + 1) & mask) {
+ if (counts[slot] == 0) {
+ bits[slot] = value;
+ docIDs[slot] = docID;
+ ++size;
+ } else if (bits[slot] != value) {
+ continue;
+ }
+ counts[slot] += count;
+ break;
+ }
+ }
+
+ private void rehash() {
+ final long[] oldBits = bits;
+ final int[] oldCounts = counts;
+ final int[] oldDocIDs = docIDs;
+
+ final int newCapacity = bits.length * 2;
+ bits = new long[newCapacity];
+ counts = new int[newCapacity];
+ docIDs = new int[newCapacity];
+ mask = newCapacity - 1;
+ threshold = (int) (LOAD_FACTOR * newCapacity);
+ size = 0;
+
+ for (int i = 0; i < oldBits.length; ++i) {
+ if (oldCounts[i] > 0) {
+ add(oldDocIDs[i], oldBits[i], oldCounts[i]);
+ }
+ }
+ }
+
+ }
+
+ private static class Entry {
+ int docID;
+ int count;
+ long bits;
+ }
+
+ public static NamedList<Integer> getCounts(SolrIndexSearcher searcher, DocSet docs, String fieldName, int offset, int limit, int mincount, boolean missing, String sort) throws IOException {
+ final boolean zeros = mincount <= 0;
+ mincount = Math.max(mincount, 1);
+ final SchemaField sf = searcher.getSchema().getField(fieldName);
+ final FieldType ft = sf.getType();
+ final NumericType numericType = ft.getNumericType();
+ if (numericType == null) {
+ throw new IllegalStateException();
+ }
+ final List<AtomicReaderContext> leaves = searcher.getIndexReader().leaves();
+
+ // 1. accumulate
+ final HashTable hashTable = new HashTable();
+ final Iterator<AtomicReaderContext> ctxIt = leaves.iterator();
+ AtomicReaderContext ctx = null;
+ FieldCache.Longs longs = null;
+ Bits docsWithField = null;
+ int missingCount = 0;
+ for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
+ final int doc = docsIt.nextDoc();
+ if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
+ do {
+ ctx = ctxIt.next();
+ } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc());
+ assert doc >= ctx.docBase;
+ switch (numericType) {
+ case LONG:
+ longs = FieldCache.DEFAULT.getLongs(ctx.reader(), fieldName, true);
+ break;
+ case INT:
+ final FieldCache.Ints ints = FieldCache.DEFAULT.getInts(ctx.reader(), fieldName, true);
+ longs = new FieldCache.Longs() {
+ @Override
+ public long get(int docID) {
+ return ints.get(docID);
+ }
+ };
+ break;
+ case FLOAT:
+ final FieldCache.Floats floats = FieldCache.DEFAULT.getFloats(ctx.reader(), fieldName, true);
+ longs = new FieldCache.Longs() {
+ @Override
+ public long get(int docID) {
+ return Float.floatToIntBits(floats.get(docID));
+ }
+ };
+ break;
+ case DOUBLE:
+ final FieldCache.Doubles doubles = FieldCache.DEFAULT.getDoubles(ctx.reader(), fieldName, true);
+ longs = new FieldCache.Longs() {
+ @Override
+ public long get(int docID) {
+ return Double.doubleToLongBits(doubles.get(docID));
+ }
+ };
+ break;
+ default:
+ throw new AssertionError();
+ }
+ docsWithField = FieldCache.DEFAULT.getDocsWithField(ctx.reader(), fieldName);
+ }
+ if (docsWithField.get(doc - ctx.docBase)) {
+ hashTable.add(doc, longs.get(doc - ctx.docBase), 1);
+ } else {
+ ++missingCount;
+ }
+ }
+
+ // 2. select top-k facet values
+ final int pqSize = limit < 0 ? hashTable.size : Math.min(offset + limit, hashTable.size);
+ final PriorityQueue<Entry> pq;
+ if (FacetParams.FACET_SORT_COUNT.equals(sort) || FacetParams.FACET_SORT_COUNT_LEGACY.equals(sort)) {
+ pq = new PriorityQueue<Entry>(pqSize) {
+ @Override
+ protected boolean lessThan(Entry a, Entry b) {
+ if (a.count < b.count || (a.count == b.count && a.bits > b.bits)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ };
+ } else {
+ pq = new PriorityQueue<Entry>(pqSize) {
+ @Override
+ protected boolean lessThan(Entry a, Entry b) {
+ return a.bits > b.bits;
+ }
+ };
+ }
+ Entry e = null;
+ for (int i = 0; i < hashTable.bits.length; ++i) {
+ if (hashTable.counts[i] >= mincount) {
+ if (e == null) {
+ e = new Entry();
+ }
+ e.bits = hashTable.bits[i];
+ e.count = hashTable.counts[i];
+ e.docID = hashTable.docIDs[i];
+ e = pq.insertWithOverflow(e);
+ }
+ }
+
+ // 4. build the NamedList
+ final ValueSource vs = ft.getValueSource(sf, null);
+ final NamedList<Integer> result = new NamedList<Integer>();
+
+ // This stuff is complicated because if facet.mincount=0, the counts needs
+ // to be merged with terms from the terms dict
+ if (!zeros || FacetParams.FACET_SORT_COUNT.equals(sort) || FacetParams.FACET_SORT_COUNT_LEGACY.equals(sort)) {
+ // Only keep items we're interested in
+ final Deque<Entry> counts = new ArrayDeque<Entry>();
+ while (pq.size() > offset) {
+ counts.addFirst(pq.pop());
+ }
+
+ // Entries from the PQ first, then using the terms dictionary
+ for (Entry entry : counts) {
+ final int readerIdx = ReaderUtil.subIndex(entry.docID, leaves);
+ final FunctionValues values = vs.getValues(Collections.emptyMap(), leaves.get(readerIdx));
+ result.add(values.strVal(entry.docID - leaves.get(readerIdx).docBase), entry.count);
+ }
+
+ if (zeros && (limit < 0 || result.size() < limit)) { // need to merge with the term dict
+ if (!sf.indexed()) {
+ throw new IllegalStateException("Cannot use " + FacetParams.FACET_MINCOUNT + "=0 on a field which is not indexed");
+ }
+ // Add zeros until there are limit results
+ final Set<String> alreadySeen = new HashSet<String>();
+ while (pq.size() > 0) {
+ Entry entry = pq.pop();
+ final int readerIdx = ReaderUtil.subIndex(entry.docID, leaves);
+ final FunctionValues values = vs.getValues(Collections.emptyMap(), leaves.get(readerIdx));
+ alreadySeen.add(values.strVal(entry.docID - leaves.get(readerIdx).docBase));
+ }
+ for (int i = 0; i < result.size(); ++i) {
+ alreadySeen.add(result.getName(i));
+ }
+ final Terms terms = searcher.getAtomicReader().terms(fieldName);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator(null);
+ BytesRef term = termsEnum.next();
+ final CharsRef spare = new CharsRef();
+ for (int skipped = hashTable.size; skipped < offset && term != null; ) {
+ ft.indexedToReadable(term, spare);
+ final String termStr = spare.toString();
+ if (!alreadySeen.contains(termStr)) {
+ ++skipped;
+ }
+ term = termsEnum.next();
+ }
+ for ( ; term != null && (limit < 0 || result.size() < limit); term = termsEnum.next()) {
+ ft.indexedToReadable(term, spare);
+ final String termStr = spare.toString();
+ if (!alreadySeen.contains(termStr)) {
+ result.add(termStr, 0);
+ }
+ }
+ }
+ }
+ } else {
+ // sort=index, mincount=0 and we have less than limit items
+ // => Merge the PQ and the terms dictionary on the fly
+ if (!sf.indexed()) {
+ throw new IllegalStateException("Cannot use " + FacetParams.FACET_SORT + "=" + FacetParams.FACET_SORT_INDEX + " on a field which is not indexed");
+ }
+ final Map<String, Integer> counts = new HashMap<String, Integer>();
+ while (pq.size() > 0) {
+ final Entry entry = pq.pop();
+ final int readerIdx = ReaderUtil.subIndex(entry.docID, leaves);
+ final FunctionValues values = vs.getValues(Collections.emptyMap(), leaves.get(readerIdx));
+ counts.put(values.strVal(entry.docID - leaves.get(readerIdx).docBase), entry.count);
+ }
+ final Terms terms = searcher.getAtomicReader().terms(fieldName);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator(null);
+ final CharsRef spare = new CharsRef();
+ BytesRef term = termsEnum.next();
+ for (int i = 0; i < offset && term != null; ++i) {
+ term = termsEnum.next();
+ }
+ for ( ; term != null && (limit < 0 || result.size() < limit); term = termsEnum.next()) {
+ ft.indexedToReadable(term, spare);
+ final String termStr = spare.toString();
+ Integer count = counts.get(termStr);
+ if (count == null) {
+ count = 0;
+ }
+ result.add(termStr, count);
+ }
+ }
+ }
+
+ if (missing) {
+ result.add(null, missingCount);
+ }
+ return result;
+ }
+
+}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/SimpleFacets.java Sat Feb 16 18:50:20 2013
@@ -17,37 +17,83 @@
package org.apache.solr.request;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.MultiDocsEnum;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.grouping.AbstractAllGroupHeadsCollector;
-import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
import org.apache.lucene.search.grouping.term.TermAllGroupsCollector;
-import org.apache.lucene.util.*;
-import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.search.grouping.term.TermGroupFacetCollector;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.params.*;
-import org.apache.solr.common.params.FacetParams.FacetRangeOther;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.FacetParams.FacetRangeInclude;
+import org.apache.solr.common.params.FacetParams.FacetRangeOther;
+import org.apache.solr.common.params.GroupParams;
+import org.apache.solr.common.params.RequiredSolrParams;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.schema.*;
-import org.apache.solr.search.*;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.schema.BoolField;
+import org.apache.solr.schema.DateField;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.SortableDoubleField;
+import org.apache.solr.schema.SortableFloatField;
+import org.apache.solr.schema.SortableIntField;
+import org.apache.solr.schema.SortableLongField;
+import org.apache.solr.schema.TrieField;
+import org.apache.solr.search.BitDocSet;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocSet;
+import org.apache.solr.search.Grouping;
+import org.apache.solr.search.HashDocSet;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.QueryParsing;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SortedIntDocSet;
+import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.grouping.GroupingSpecification;
import org.apache.solr.util.BoundedTreeSet;
import org.apache.solr.util.DateMathParser;
import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.util.LongPriorityQueue;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.Executor;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
/**
* A class that generates simple Facet information for a request.
*
@@ -300,7 +346,8 @@ public class SimpleFacets {
boolean enumMethod = FacetParams.FACET_METHOD_enum.equals(method);
// TODO: default to per-segment or not?
- boolean per_segment = FacetParams.FACET_METHOD_fcs.equals(method);
+ boolean per_segment = FacetParams.FACET_METHOD_fcs.equals(method) // explicit
+ || (ft.getNumericType() != null && sf.hasDocValues()); // numeric doc values are per-segment by default
if (method == null && ft instanceof BoolField) {
// Always use filters for booleans... we know the number of values is very small.
@@ -329,10 +376,18 @@ public class SimpleFacets {
// TODO: future logic could use filters instead of the fieldcache if
// the number of terms in the field is small enough.
if (per_segment) {
- PerSegmentSingleValuedFaceting ps = new PerSegmentSingleValuedFaceting(searcher, docs, field, offset,limit, mincount, missing, sort, prefix);
- Executor executor = threads == 0 ? directExecutor : facetExecutor;
- ps.setNumThreads(threads);
- counts = ps.getFacetCounts(executor);
+ if (ft.getNumericType() != null && !sf.multiValued()) {
+ // force numeric faceting
+ if (prefix != null && !prefix.isEmpty()) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, FacetParams.FACET_PREFIX + " is not supported on numeric types");
+ }
+ counts = NumericFacets.getCounts(searcher, docs, field, offset, limit, mincount, missing, sort);
+ } else {
+ PerSegmentSingleValuedFaceting ps = new PerSegmentSingleValuedFaceting(searcher, docs, field, offset,limit, mincount, missing, sort, prefix);
+ Executor executor = threads == 0 ? directExecutor : facetExecutor;
+ ps.setNumThreads(threads);
+ counts = ps.getFacetCounts(executor);
+ }
} else {
counts = getFieldCacheCounts(searcher, docs, field, offset,limit, mincount, missing, sort, prefix);
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/request/UnInvertedField.java Sat Feb 16 18:50:20 2013
@@ -483,13 +483,7 @@ public class UnInvertedField extends Doc
SortedDocValues si;
for (String f : facet) {
SchemaField facet_sf = searcher.getSchema().getField(f);
- try {
- si = FieldCache.DEFAULT.getTermsIndex(searcher.getAtomicReader(), f);
- }
- catch (IOException e) {
- throw new RuntimeException("failed to open field cache for: " + f, e);
- }
- finfo[i] = new FieldFacetStats(f, si, sf, facet_sf, numTermsInField);
+ finfo[i] = new FieldFacetStats(searcher, f, sf, facet_sf);
i++;
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/AbstractSpatialFieldType.java Sat Feb 16 18:50:20 2013
@@ -51,6 +51,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -103,7 +107,7 @@ public abstract class AbstractSpatialFie
}
@Override
- public Field[] createFields(SchemaField field, Object val, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object val, float boost) {
String shapeStr = null;
Shape shape = null;
if (val instanceof Shape) {
@@ -114,34 +118,22 @@ public abstract class AbstractSpatialFie
}
if (shape == null) {
log.debug("Field {}: null shape for input: {}", field, val);
- return null;
+ return Collections.emptyList();
}
- Field[] indexableFields = null;
+ List<StorableField> result = new ArrayList<StorableField>();
if (field.indexed()) {
T strategy = getStrategy(field.getName());
- indexableFields = strategy.createIndexableFields(shape);
+ result.addAll(Arrays.asList(strategy.createIndexableFields(shape)));
}
- StoredField storedField = null;
if (field.stored()) {
if (shapeStr == null)
shapeStr = shapeToString(shape);
- storedField = new StoredField(field.getName(), shapeStr);
+ result.add(new StoredField(field.getName(), shapeStr));
}
- if (indexableFields == null) {
- if (storedField == null)
- return null;
- return new Field[]{storedField};
- } else {
- if (storedField == null)
- return indexableFields;
- Field[] result = new Field[indexableFields.length+1];
- System.arraycopy(indexableFields,0,result,0,indexableFields.length);
- result[result.length-1] = storedField;
- return result;
- }
+ return result;
}
protected Shape parseShape(String shapeStr) {
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CurrencyField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CurrencyField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CurrencyField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CurrencyField.java Sat Feb 16 18:50:20 2013
@@ -46,9 +46,11 @@ import javax.xml.xpath.XPathExpressionEx
import javax.xml.xpath.XPathFactory;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
import java.util.Currency;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -145,14 +147,14 @@ public class CurrencyField extends Field
}
@Override
- public StorableField[] createFields(SchemaField field, Object externalVal, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object externalVal, float boost) {
CurrencyValue value = CurrencyValue.parse(externalVal.toString(), defaultCurrency);
- StorableField[] f = new StorableField[field.stored() ? 3 : 2];
+ List<StorableField> f = new ArrayList<StorableField>();
SchemaField amountField = getAmountField(field);
- f[0] = amountField.createField(String.valueOf(value.getAmount()), amountField.indexed() && !amountField.omitNorms() ? boost : 1F);
+ f.add(amountField.createField(String.valueOf(value.getAmount()), amountField.indexed() && !amountField.omitNorms() ? boost : 1F));
SchemaField currencyField = getCurrencyField(field);
- f[1] = currencyField.createField(value.getCurrencyCode(), currencyField.indexed() && !currencyField.omitNorms() ? boost : 1F);
+ f.add(currencyField.createField(value.getCurrencyCode(), currencyField.indexed() && !currencyField.omitNorms() ? boost : 1F));
if (field.stored()) {
org.apache.lucene.document.FieldType customType = new org.apache.lucene.document.FieldType();
@@ -162,7 +164,7 @@ public class CurrencyField extends Field
if (storedValue.indexOf(",") < 0) {
storedValue += "," + defaultCurrency;
}
- f[2] = createField(field.getName(), storedValue, customType, 1F);
+ f.add(createField(field.getName(), storedValue, customType, 1F));
}
return f;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/DateField.java Sat Feb 16 18:50:20 2013
@@ -435,7 +435,7 @@ public class DateField extends Primitive
@Override
public ValueSource getValueSource(SchemaField field, QParser parser) {
field.checkFieldCacheSource(parser);
- return new DateFieldSource(field.getName(), field.getType());
+ return new DateFieldSource(field.getName(), field);
}
/** DateField specific range query */
@@ -453,11 +453,13 @@ public class DateField extends Primitive
class DateFieldSource extends FieldCacheSource {
// NOTE: this is bad for serialization... but we currently need the fieldType for toInternal()
+ SchemaField sf;
FieldType ft;
- public DateFieldSource(String name, FieldType ft) {
+ public DateFieldSource(String name, SchemaField sf) {
super(name);
- this.ft = ft;
+ this.sf = sf;
+ this.ft = sf.getType();
}
@Override
@@ -475,6 +477,11 @@ class DateFieldSource extends FieldCache
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public float floatVal(int doc) {
return (float)intVal(doc);
}
@@ -514,7 +521,7 @@ class DateFieldSource extends FieldCache
} else {
final BytesRef br = new BytesRef();
termsIndex.lookupOrd(ord, br);
- return ft.toObject(null, br);
+ return ft.toObject(sf, br);
}
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldProperties.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldProperties.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldProperties.java Sat Feb 16 18:50:20 2013
@@ -50,6 +50,7 @@ public abstract class FieldProperties {
protected final static int OMIT_POSITIONS = 0x00002000;
protected final static int STORE_OFFSETS = 0x00004000;
+ protected final static int DOC_VALUES = 0x00008000;
static final String[] propertyNames = {
"indexed", "tokenized", "stored",
@@ -57,7 +58,7 @@ public abstract class FieldProperties {
"termVectors", "termPositions", "termOffsets",
"multiValued",
"sortMissingFirst","sortMissingLast","required", "omitPositions",
- "storeOffsetsWithPositions"
+ "storeOffsetsWithPositions", "docValues"
};
static final Map<String,Integer> propertyMap = new HashMap<String,Integer>();
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java Sat Feb 16 18:50:20 2013
@@ -17,14 +17,20 @@
package org.apache.solr.schema;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.GeneralField;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.ValueSource;
@@ -45,11 +51,6 @@ import org.apache.solr.search.Sorting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.Reader;
-import java.util.HashMap;
-import java.util.Map;
-
/**
* Base class for all field types used by an index schema.
*
@@ -120,14 +121,6 @@ public abstract class FieldType extends
}
- protected String getArg(String n, Map<String,String> args) {
- String s = args.remove(n);
- if (s == null) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Missing parameter '"+n+"' for FieldType=" + typeName +args);
- }
- return s;
- }
-
// Handle additional arguments...
void setArgs(IndexSchema schema, Map<String,String> args) {
// default to STORED, INDEXED, OMIT_TF_POSITIONS and MULTIVALUED depending on schema version
@@ -169,11 +162,8 @@ public abstract class FieldType extends
initArgs.remove("positionIncrementGap");
}
- final String postingsFormat = initArgs.get("postingsFormat");
- if (postingsFormat != null) {
- this.postingsFormat = postingsFormat;
- initArgs.remove("postingsFormat");
- }
+ this.postingsFormat = initArgs.remove("postingsFormat");
+ this.docValuesFormat = initArgs.remove("docValuesFormat");
if (initArgs.size() > 0) {
throw new RuntimeException("schema fieldtype " + typeName
@@ -261,7 +251,7 @@ public abstract class FieldType extends
newType.setStoreTermVectors(field.storeTermVector());
newType.setStoreTermVectorOffsets(field.storeTermOffsets());
newType.setStoreTermVectorPositions(field.storeTermPositions());
-
+
return createField(field.getName(), val, newType, boost);
}
@@ -290,9 +280,15 @@ public abstract class FieldType extends
* @see #createField(SchemaField, Object, float)
* @see #isPolyField()
*/
- public StorableField[] createFields(SchemaField field, Object value, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object value, float boost) {
StorableField f = createField( field, value, boost);
- return f==null ? new StorableField[]{} : new StorableField[]{f};
+ if (field.hasDocValues() && f.fieldType().docValueType() == null) {
+ // field types that support doc values should either override createField
+ // to return a field with doc values or extend createFields if this can't
+ // be done in a single field instance (see StrField for example)
+ throw new UnsupportedOperationException("This field type does not support doc values: " + this);
+ }
+ return f==null ? Collections.<StorableField>emptyList() : Collections.singletonList(f);
}
protected IndexOptions getIndexOptions(SchemaField field, String internalVal) {
@@ -513,7 +509,13 @@ public abstract class FieldType extends
public Similarity getSimilarity() {
return similarity;
}
-
+
+ /** Return the numeric type of this field, or null if this field is not a
+ * numeric field. */
+ public org.apache.lucene.document.FieldType.NumericType getNumericType() {
+ return null;
+ }
+
/**
* Sets the Similarity used when scoring fields of this type
* @lucene.internal
@@ -530,7 +532,16 @@ public abstract class FieldType extends
public String getPostingsFormat() {
return postingsFormat;
}
-
+
+ /**
+ * The docvalues format used for this field type
+ */
+ protected String docValuesFormat;
+
+ public final String getDocValuesFormat() {
+ return docValuesFormat;
+ }
+
/**
* calls back to TextResponseWriter to write the field value
*/
@@ -562,7 +573,6 @@ public abstract class FieldType extends
return new StrFieldSource(field.name);
}
-
/**
* Returns a Query instance for doing range searches on this field type. {@link org.apache.solr.search.SolrQueryParser}
* currently passes part1 and part2 as null if they are '*' respectively. minInclusive and maxInclusive are both true
@@ -615,7 +625,11 @@ public abstract class FieldType extends
* if invariants are violated by the <code>SchemaField.</code>
* </p>
*/
- public void checkSchemaField(final SchemaField field) throws SolrException {
- // :NOOP:
+ public void checkSchemaField(final SchemaField field) {
+ // override if your field type supports doc values
+ if (field.hasDocValues()) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values");
+ }
}
+
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/LatLonType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/LatLonType.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/LatLonType.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/LatLonType.java Sat Feb 16 18:50:20 2013
@@ -16,30 +16,42 @@ package org.apache.solr.schema;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.VectorValueSource;
-import org.apache.lucene.search.*;
-import com.spatial4j.core.io.ParseUtils;
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.exception.InvalidShapeException;
-import com.spatial4j.core.shape.Rectangle;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ComplexExplanation;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
-import org.apache.solr.search.*;
-import org.apache.solr.search.function.distance.HaversineConstFunction;
+import org.apache.solr.search.DelegatingCollector;
+import org.apache.solr.search.ExtendedQueryBase;
+import org.apache.solr.search.PostFilter;
+import org.apache.solr.search.QParser;
+import org.apache.solr.search.SpatialOptions;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.distance.DistanceUtils;
+import com.spatial4j.core.exception.InvalidShapeException;
+import com.spatial4j.core.io.ParseUtils;
+import com.spatial4j.core.shape.Rectangle;
/**
@@ -57,10 +69,10 @@ public class LatLonType extends Abstract
}
@Override
- public StorableField[] createFields(SchemaField field, Object value, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object value, float boost) {
String externalVal = value.toString();
//we could have tileDiff + 3 fields (two for the lat/lon, one for storage)
- StorableField[] f = new StorableField[(field.indexed() ? 2 : 0) + (field.stored() ? 1 : 0)];
+ List<StorableField> f = new ArrayList<StorableField>(3);
if (field.indexed()) {
int i = 0;
double[] latLon;
@@ -71,18 +83,18 @@ public class LatLonType extends Abstract
}
//latitude
SchemaField lat = subField(field, i);
- f[i] = lat.createField(String.valueOf(latLon[LAT]), lat.indexed() && !lat.omitNorms() ? boost : 1f);
+ f.add(lat.createField(String.valueOf(latLon[LAT]), lat.indexed() && !lat.omitNorms() ? boost : 1f));
i++;
//longitude
SchemaField lon = subField(field, i);
- f[i] = lon.createField(String.valueOf(latLon[LON]), lon.indexed() && !lon.omitNorms() ? boost : 1f);
+ f.add(lon.createField(String.valueOf(latLon[LON]), lon.indexed() && !lon.omitNorms() ? boost : 1f));
}
if (field.stored()) {
FieldType customType = new FieldType();
customType.setStored(true);
- f[f.length - 1] = createField(field.getName(), externalVal, customType, 1f);
+ f.add(createField(field.getName(), externalVal, customType, 1f));
}
return f;
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PointType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PointType.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PointType.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PointType.java Sat Feb 16 18:50:20 2013
@@ -69,7 +69,7 @@ public class PointType extends Coordinat
}
@Override
- public StorableField[] createFields(SchemaField field, Object value, float boost) {
+ public List<StorableField> createFields(SchemaField field, Object value, float boost) {
String externalVal = value.toString();
String[] point = new String[0];
try {
@@ -79,12 +79,12 @@ public class PointType extends Coordinat
}
// TODO: this doesn't currently support polyFields as sub-field types
- StorableField[] f = new StorableField[ (field.indexed() ? dimension : 0) + (field.stored() ? 1 : 0) ];
+ List<StorableField> f = new ArrayList<StorableField>(dimension+1);
if (field.indexed()) {
for (int i=0; i<dimension; i++) {
SchemaField sf = subField(field, i);
- f[i] = sf.createField(point[i], sf.indexed() && !sf.omitNorms() ? boost : 1f);
+ f.add(sf.createField(point[i], sf.indexed() && !sf.omitNorms() ? boost : 1f));
}
}
@@ -92,7 +92,7 @@ public class PointType extends Coordinat
String storedVal = externalVal; // normalize or not?
FieldType customType = new FieldType();
customType.setStored(true);
- f[f.length - 1] = createField(field.getName(), storedVal, customType, 1f);
+ f.add(createField(field.getName(), storedVal, customType, 1f));
}
return f;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SchemaField.java Sat Feb 16 18:50:20 2013
@@ -18,13 +18,13 @@
package org.apache.solr.schema;
import org.apache.solr.common.SolrException;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.QParser;
import org.apache.solr.response.TextResponseWriter;
+import java.util.List;
import java.util.Map;
import java.io.IOException;
@@ -79,6 +79,7 @@ public final class SchemaField extends F
public boolean indexed() { return (properties & INDEXED)!=0; }
public boolean stored() { return (properties & STORED)!=0; }
+ public boolean hasDocValues() { return (properties & DOC_VALUES) != 0; }
public boolean storeTermVector() { return (properties & STORE_TERMVECTORS)!=0; }
public boolean storeTermPositions() { return (properties & STORE_TERMPOSITIONS)!=0; }
public boolean storeTermOffsets() { return (properties & STORE_TERMOFFSETS)!=0; }
@@ -104,8 +105,8 @@ public final class SchemaField extends F
public StorableField createField(Object val, float boost) {
return type.createField(this,val,boost);
}
-
- public StorableField[] createFields(Object val, float boost) {
+
+ public List<StorableField> createFields(Object val, float boost) {
return type.createFields(this,val,boost);
}
@@ -148,9 +149,9 @@ public final class SchemaField extends F
* @see FieldType#getSortField
*/
public void checkSortability() throws SolrException {
- if (! indexed() ) {
+ if (! (indexed() || hasDocValues()) ) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "can not sort on unindexed field: "
+ "can not sort on a field which is neither indexed nor has doc values: "
+ getName());
}
if ( multiValued() ) {
@@ -169,9 +170,9 @@ public final class SchemaField extends F
* @see FieldType#getValueSource
*/
public void checkFieldCacheSource(QParser parser) throws SolrException {
- if (! indexed() ) {
+ if (! (indexed() || hasDocValues()) ) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "can not use FieldCache on unindexed field: "
+ "can not use FieldCache on a field which is neither indexed nor has doc values: "
+ getName());
}
if ( multiValued() ) {
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java Sat Feb 16 18:50:20 2013
@@ -132,6 +132,11 @@ class SortableDoubleFieldSource extends
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public float floatVal(int doc) {
return (float)doubleVal(doc);
}
@@ -164,13 +169,7 @@ class SortableDoubleFieldSource extends
@Override
public Object objectVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- return null;
- } else {
- termsIndex.lookupOrd(ord, spare);
- return NumberUtils.SortableStr2double(spare);
- }
+ return exists(doc) ? doubleVal(doc) : null;
}
@Override
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java Sat Feb 16 18:50:20 2013
@@ -136,6 +136,11 @@ class SortableFloatFieldSource extends F
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public float floatVal(int doc) {
int ord=termsIndex.getOrd(doc);
if (ord==-1) {
@@ -173,13 +178,7 @@ class SortableFloatFieldSource extends F
@Override
public Object objectVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- return null;
- } else {
- termsIndex.lookupOrd(ord, spare);
- return NumberUtils.SortableStr2float(spare);
- }
+ return exists(doc) ? floatVal(doc) : null;
}
@Override
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java Sat Feb 16 18:50:20 2013
@@ -143,6 +143,11 @@ class SortableIntFieldSource extends Fie
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public int intVal(int doc) {
int ord=termsIndex.getOrd(doc);
if (ord==-1) {
@@ -175,13 +180,7 @@ class SortableIntFieldSource extends Fie
@Override
public Object objectVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- return null;
- } else {
- termsIndex.lookupOrd(ord, spare);
- return NumberUtils.SortableStr2int(spare);
- }
+ return exists(doc) ? intVal(doc) : null;
}
@Override
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java Sat Feb 16 18:50:20 2013
@@ -136,6 +136,11 @@ class SortableLongFieldSource extends Fi
}
@Override
+ public boolean exists(int doc) {
+ return termsIndex.getOrd(doc) >= 0;
+ }
+
+ @Override
public float floatVal(int doc) {
return (float)longVal(doc);
}
@@ -168,13 +173,7 @@ class SortableLongFieldSource extends Fi
@Override
public Object objectVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- return null;
- } else {
- termsIndex.lookupOrd(ord, spare);
- return NumberUtils.SortableStr2long(spare);
- }
+ return exists(doc) ? longVal(doc) : null;
}
@Override
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java Sat Feb 16 18:50:20 2013
@@ -17,20 +17,43 @@
package org.apache.solr.schema;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.index.GeneralField;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.StorableField;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
-import java.io.IOException;
-/**
- *
- */
public class StrField extends PrimitiveFieldType {
+
+ @Override
+ protected void init(IndexSchema schema, Map<String,String> args) {
+ super.init(schema, args);
+ }
+
+ @Override
+ public List<StorableField> createFields(SchemaField field, Object value,
+ float boost) {
+ if (field.hasDocValues()) {
+ List<StorableField> fields = new ArrayList<StorableField>();
+ fields.add(createField(field, value, boost));
+ final BytesRef bytes = new BytesRef(value.toString());
+ final Field docValuesField = new SortedDocValuesField(field.getName(), bytes);
+ fields.add(docValuesField);
+ return fields;
+ } else {
+ return Collections.singletonList(createField(field, value, boost));
+ }
+ }
+
@Override
public SortField getSortField(SchemaField field,boolean reverse) {
return getStringSort(field,reverse);
@@ -51,6 +74,14 @@ public class StrField extends PrimitiveF
public Object toObject(SchemaField sf, BytesRef term) {
return term.utf8ToString();
}
+
+ @Override
+ public void checkSchemaField(SchemaField field) {
+ // change me when multi-valued doc values are supported
+ if (field.hasDocValues() && !(field.isRequired() || field.getDefaultValue() != null)) {
+ throw new IllegalStateException("Field " + this + " has doc values enabled, but has no default value and is not required");
+ }
+ }
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieDateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieDateField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieDateField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieDateField.java Sat Feb 16 18:50:20 2013
@@ -17,11 +17,13 @@
package org.apache.solr.schema;
+import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.solr.search.QParser;
+import org.apache.solr.common.SolrException;
import org.apache.solr.response.TextResponseWriter;
-import org.apache.lucene.index.GeneralField;
-import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.document.FieldType.NumericType;
+import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.Query;
@@ -29,6 +31,7 @@ import org.apache.lucene.search.NumericR
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
+import java.util.List;
import java.util.Map;
import java.util.Date;
import java.io.IOException;
@@ -73,6 +76,10 @@ public class TrieDateField extends DateF
return wrappedField.getPrecisionStep();
}
+ @Override
+ public NumericType getNumericType() {
+ return wrappedField.getNumericType();
+ }
@Override
public void write(TextResponseWriter writer, String name, StorableField f) throws IOException {
@@ -130,6 +137,11 @@ public class TrieDateField extends DateF
}
@Override
+ public List<StorableField> createFields(SchemaField field, Object value, float boost) {
+ return wrappedField.createFields(field, value, boost);
+ }
+
+ @Override
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
return wrappedField.getRangeQuery(parser, field, min, max, minInclusive, maxInclusive);
}
@@ -141,4 +153,10 @@ public class TrieDateField extends DateF
max == null ? null : max.getTime(),
minInclusive, maxInclusive);
}
+
+ @Override
+ public void checkSchemaField(SchemaField field) {
+ wrappedField.checkSchemaField(field);
+ }
+
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TrieField.java Sat Feb 16 18:50:20 2013
@@ -17,7 +17,10 @@
package org.apache.solr.schema;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -29,13 +32,17 @@ import org.apache.lucene.document.FieldT
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleFieldSource;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
-import org.apache.lucene.search.*;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.NumericUtils;
@@ -101,8 +108,7 @@ public class TrieField extends Primitive
"Invalid type specified in schema.xml for field: " + args.get("name"), e);
}
}
-
-
+
CharFilterFactory[] filterFactories = new CharFilterFactory[0];
TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0];
analyzer = new TokenizerChain(filterFactories, new TrieTokenizerFactory(type, precisionStep), tokenFilterFactories);
@@ -237,6 +243,23 @@ public class TrieField extends Primitive
}
@Override
+ public NumericType getNumericType() {
+ switch (type) {
+ case INTEGER:
+ return NumericType.INT;
+ case LONG:
+ case DATE:
+ return NumericType.LONG;
+ case FLOAT:
+ return NumericType.FLOAT;
+ case DOUBLE:
+ return NumericType.DOUBLE;
+ default:
+ throw new AssertionError();
+ }
+ }
+
+ @Override
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
int ps = precisionStep;
Query query = null;
@@ -473,8 +496,9 @@ public class TrieField extends Primitive
public StorableField createField(SchemaField field, Object value, float boost) {
boolean indexed = field.indexed();
boolean stored = field.stored();
+ boolean docValues = field.hasDocValues();
- if (!indexed && !stored) {
+ if (!indexed && !stored && !docValues) {
if (log.isTraceEnabled())
log.trace("Ignoring unindexed/unstored field: " + field);
return null;
@@ -549,6 +573,28 @@ public class TrieField extends Primitive
return f;
}
+ @Override
+ public List<StorableField> createFields(SchemaField sf, Object value, float boost) {
+ if (sf.hasDocValues()) {
+ List<StorableField> fields = new ArrayList<StorableField>();
+ final StorableField field = createField(sf, value, boost);
+ fields.add(field);
+ final long bits;
+ if (field.numericValue() instanceof Integer || field.numericValue() instanceof Long) {
+ bits = field.numericValue().longValue();
+ } else if (field.numericValue() instanceof Float) {
+ bits = Float.floatToIntBits(field.numericValue().floatValue());
+ } else {
+ assert field.numericValue() instanceof Double;
+ bits = Double.doubleToLongBits(field.numericValue().doubleValue());
+ }
+ fields.add(new NumericDocValuesField(sf.getName(), bits));
+ return fields;
+ } else {
+ return Collections.singletonList(createField(sf, value, boost));
+ }
+ }
+
public enum TrieTypes {
INTEGER,
LONG,
@@ -586,6 +632,13 @@ public class TrieField extends Primitive
}
return null;
}
+
+ @Override
+ public void checkSchemaField(final SchemaField field) {
+ if (field.hasDocValues() && !(field.isRequired() || field.getDefaultValue() != null)) {
+ throw new IllegalStateException("Field " + this + " has doc values enabled, but has no default value and is not required");
+ }
+ }
}
class TrieDateFieldSource extends LongFieldSource {
@@ -605,14 +658,20 @@ class TrieDateFieldSource extends LongFi
}
@Override
- public Object longToObject(long val) {
+ public Date longToObject(long val) {
return new Date(val);
}
@Override
+ public String longToString(long val) {
+ return TrieField.dateField.toExternal(longToObject(val));
+ }
+
+ @Override
public long externalToLong(String extVal) {
return TrieField.dateField.parseMath(null, extVal).getTime();
}
+
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/UUIDField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/UUIDField.java?rev=1446922&r1=1446921&r2=1446922&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/UUIDField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/UUIDField.java Sat Feb 16 18:50:20 2013
@@ -22,8 +22,6 @@ import java.util.Locale;
import java.util.Map;
import java.util.UUID;
-import org.apache.lucene.index.GeneralField;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.SortField;
import org.apache.solr.common.SolrException;