You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by yo...@apache.org on 2009/08/03 23:09:06 UTC
svn commit: r800565 - in /lucene/solr/trunk: ./ example/solr/conf/
src/java/org/apache/solr/request/ src/java/org/apache/solr/schema/
src/java/org/apache/solr/search/ src/java/org/apache/solr/search/function/
src/java/org/apache/solr/util/ src/test/org...
Author: yonik
Date: Mon Aug 3 21:09:05 2009
New Revision: 800565
URL: http://svn.apache.org/viewvc?rev=800565&view=rev
Log:
SOLR-1291 SOLR-1322 SOLR-1325 SOLR-1328 : more trie integration
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/example/solr/conf/schema.xml
lucene/solr/trunk/src/java/org/apache/solr/request/BinaryResponseWriter.java
lucene/solr/trunk/src/java/org/apache/solr/request/SimpleFacets.java
lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java
lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java
lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java
lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java
lucene/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java
lucene/solr/trunk/src/java/org/apache/solr/search/function/LongFieldSource.java
lucene/solr/trunk/src/java/org/apache/solr/util/AbstractSolrTestCase.java
lucene/solr/trunk/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
lucene/solr/trunk/src/test/org/apache/solr/search/TestQueryTypes.java
lucene/solr/trunk/src/test/org/apache/solr/search/TestRangeQuery.java
lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml
lucene/solr/trunk/src/test/test-files/solr/conf/schema11.xml
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Mon Aug 3 21:09:05 2009
@@ -180,10 +180,10 @@
and lucene query parser (the latter existed as an undocumented
feature in 1.3) (yonik)
-40. SOLR-940: Add support for Lucene's Trie Range Queries by providing new FieldTypes in schema for int, float, long,
- double and date. Range searches and term queries on such fields will automatically use the corresponding trie
- range filter in Lucene contrib-queries and can be dramatically faster than normal range queries.
- (Uwe Schindler, shalin)
+40. SOLR-940: Add support for Lucene's Trie Range Queries by providing new FieldTypes in
+ schema for int, float, long, double and date. Single-valued Trie based
+ fields with a precisionStep will index multiple precisions and enable
+ faster range queries. (Uwe Schindler, yonik, shalin)
41. SOLR-1038: Enhance CommonsHttpSolrServer to add docs in batch using an iterator API (Noble Paul via shalin)
Modified: lucene/solr/trunk/example/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/example/solr/conf/schema.xml?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/example/solr/conf/schema.xml (original)
+++ lucene/solr/trunk/example/solr/conf/schema.xml Mon Aug 3 21:09:05 2009
@@ -63,6 +63,7 @@
<!-- The optional sortMissingLast and sortMissingFirst attributes are
currently supported on types that are sorted internally as strings.
+ This includes "string","boolean","sint","slong","sfloat","sdouble","pdate"
- If sortMissingLast="true", then a sort on this field will cause documents
without the field to come after documents with the field,
regardless of the requested sort order (asc or desc).
@@ -74,25 +75,31 @@
field first in an ascending sort and last in a descending sort.
-->
+ <!-- Default numeric field types. For faster range queries, use the tint/tfloat/tlong/tdouble types.
+ Note: the statistics component does not yet work with these field types.
+ -->
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
- <!-- numeric field types that store and index the text
- value verbatim (and hence don't support range queries, since the
- lexicographic ordering isn't equal to the numeric ordering) -->
- <fieldType name="integer" class="solr.IntField" omitNorms="true"/>
- <fieldType name="long" class="solr.LongField" omitNorms="true"/>
- <fieldType name="float" class="solr.FloatField" omitNorms="true"/>
- <fieldType name="double" class="solr.DoubleField" omitNorms="true"/>
-
-
- <!-- Numeric field types that manipulate the value into
- a string value that isn't human-readable in its internal form,
- but with a lexicographic ordering the same as the numeric ordering,
- so that range queries work correctly. -->
- <fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
+ <!--
+ Numeric field types for single-valued fields that index extra tokens with
+ lower precision to accelerate range queries when the number of values between
+ the range endpoints is large. See the javadoc for NumericRangeQuery for
+ internal implementation details.
+
+ For single-valued fields, smaller precisionStep values (specified in bits)
+ will lead to more tokens indexed per value, slightly higher index size, and
+ faster range queries.
+ Note: precisionStep is disabled for multiValued fields.
+ Note: faceting does not currently work for these fields.
+ -->
+ <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
@@ -114,35 +121,33 @@
Consult the DateField javadocs for more information.
-->
- <fieldType name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
- <!--
- Numeric field types that manipulate the value into trie encoded strings which are not
- human readable in the internal form. Range searches on such fields use the fast Trie Range Queries
- which are much faster than range searches on the SortableNumberField types.
-
- For the fast range search to work, trie fields must be indexed.
-
- For each number being added to this field, multiple terms are generated as per the algorithm described in
- org.apache.lucene.search.trie package description. The possible number of terms depend on the precisionStep
- attribute and increase dramatically with higher precision steps (factor 2**precisionStep). The default
- value of precisionStep is 8.
-
- Note that if you use a precisionStep of 32 for int/float and 64 for long/double, then multiple terms
- will not be generated, and range search will be no faster than any other number field.
- -->
- <fieldType name="tint" class="solr.TrieField" type="integer" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tfloat" class="solr.TrieField" type="float" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tlong" class="solr.TrieField" type="long" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tdouble" class="solr.TrieField" type="double" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <!-- A Trie based single-valued date field for faster date range queries and date faceting -->
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="8" positionIncrementGap="0"/>
+
+
+ <!-- plain numeric field types that store and index the text
+ value verbatim (and hence don't support range queries, since the
+ lexicographic ordering isn't equal to the numeric ordering)
+ These should only be used for compatibility with existing indexes.
+ Use Trie based fields instead.
+ -->
+ <fieldType name="pint" class="solr.IntField" omitNorms="true"/>
+ <fieldType name="plong" class="solr.LongField" omitNorms="true"/>
+ <fieldType name="pfloat" class="solr.FloatField" omitNorms="true"/>
+ <fieldType name="pdouble" class="solr.DoubleField" omitNorms="true"/>
+ <fieldType name="pdate" class="solr.DateField" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="tdouble4" class="solr.TrieField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<!--
- This date field manipulates the value into a trie encoded strings for fast range searches. They follow the
- same format and semantics as the normal DateField and support the date math syntax.
- -->
- <fieldType name="tdate" class="solr.TrieField" type="date" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ These types should only be used for back compatibility with existing
+ indexes, or if "sortMissingLast" funcitonallity is needed. Use Trie based fields instead.
+ -->
+ <fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/>
+ <fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
<!-- The "RandomSortField" is not used to store or search any
@@ -356,17 +361,17 @@
<field name="features" type="text" indexed="true" stored="true" multiValued="true" termVectors="true" termPositions="true" termOffsets="true"/>
<field name="includes" type="text" indexed="true" stored="true"/>
- <field name="weight" type="sfloat" indexed="true" stored="true"/>
- <field name="price" type="sfloat" indexed="true" stored="true"/>
+ <field name="weight" type="float" indexed="true" stored="true"/>
+ <field name="price" type="float" indexed="true" stored="true"/>
<!-- "default" values can be specified for fields, indicating which
value should be used if no value is specified when adding a document.
-->
- <field name="popularity" type="sint" indexed="true" stored="true"/>
+ <field name="popularity" type="int" indexed="true" stored="true"/>
<field name="inStock" type="boolean" indexed="true" stored="true"/>
<!-- Some sample docs exists solely to demonstrate the spellchecker
functionality, this is the only field they contain.
- Typically you might build the spellchecker of "catchall" type field
+ Typically you might build the spellchecker off "catchall" type field
containing all of the text in each document.
-->
<field name="word" type="string" indexed="true" stored="true"/>
@@ -389,17 +394,6 @@
<field name="spell" type="textSpell" indexed="true" stored="true" multiValued="true"/>
- <!-- Some examples of trie fields -->
- <field name="tint" type="tint" indexed="true" stored="false" />
- <field name="tfloat" type="tfloat" indexed="true" stored="false" />
- <field name="tlong" type="tlong" indexed="true" stored="false" />
- <field name="tdouble" type="tdouble" indexed="true" stored="false" />
-
- <!-- A double with a custom precisionStep -->
- <field name="tdouble4" type="tdouble4" indexed="true" stored="false" />
-
- <!-- An example for the trie date field -->
- <field name="tdate" type="tdate" indexed="true" stored="true" />
<!-- Dynamic field definitions. If a field name is not found, dynamicFields
will be used if the name matches any of the patterns.
@@ -408,19 +402,21 @@
EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
Longer patterns will be matched first. if equal size patterns
both match, the first appearing in the schema will be used. -->
- <dynamicField name="*_i" type="sint" indexed="true" stored="true"/>
+ <dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
- <dynamicField name="*_l" type="slong" indexed="true" stored="true"/>
+ <dynamicField name="*_l" type="long" indexed="true" stored="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
- <dynamicField name="*_f" type="sfloat" indexed="true" stored="true"/>
- <dynamicField name="*_d" type="sdouble" indexed="true" stored="true"/>
+ <dynamicField name="*_f" type="float" indexed="true" stored="true"/>
+ <dynamicField name="*_d" type="double" indexed="true" stored="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
- <dynamicField name="ignored_*" type="ignored"/>
+ <dynamicField name="*_pi" type="pint" indexed="true" stored="true"/>
+
+ <dynamicField name="ignored_*" type="ignored" multiValued="true"/>
<dynamicField name="attr_*" type="text" indexed="true" stored="true" multiValued="true"/>
- <dynamicField name="random*" type="random" />
+ <dynamicField name="random_*" type="random" />
<!-- uncomment the following to ignore any fields that don't already match an existing
field name or dynamic field, rather than reporting them as an error.
Modified: lucene/solr/trunk/src/java/org/apache/solr/request/BinaryResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/request/BinaryResponseWriter.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/request/BinaryResponseWriter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/request/BinaryResponseWriter.java Mon Aug 3 21:09:05 2009
@@ -211,6 +211,11 @@
KNOWN_TYPES.add(StrField.class);
KNOWN_TYPES.add(TextField.class);
KNOWN_TYPES.add(TrieField.class);
+ KNOWN_TYPES.add(TrieIntField.class);
+ KNOWN_TYPES.add(TrieLongField.class);
+ KNOWN_TYPES.add(TrieFloatField.class);
+ KNOWN_TYPES.add(TrieDoubleField.class);
+ KNOWN_TYPES.add(TrieDateField.class);
KNOWN_TYPES.add(BinaryField.class);
// We do not add UUIDField because UUID object is not a supported type in JavaBinCodec
// and if we write UUIDField.toObject, we wouldn't know how to handle it in the client side
Modified: lucene/solr/trunk/src/java/org/apache/solr/request/SimpleFacets.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/request/SimpleFacets.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/request/SimpleFacets.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/request/SimpleFacets.java Mon Aug 3 21:09:05 2009
@@ -33,11 +33,7 @@
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.BoolField;
-import org.apache.solr.schema.DateField;
+import org.apache.solr.schema.*;
import org.apache.solr.search.*;
import org.apache.solr.util.BoundedTreeSet;
import org.apache.solr.util.DateMathParser;
@@ -556,13 +552,13 @@
final NamedList resInner = new SimpleOrderedMap();
resOuter.add(key, resInner);
- final FieldType trash = schema.getFieldType(f);
- if (! (trash instanceof DateField)) {
+ final SchemaField sf = schema.getField(f);
+ if (! (sf.getType() instanceof DateField)) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
"Can not date facet on a field which is not a DateField: " + f);
}
- final DateField ft = (DateField) trash;
+ final DateField ft = (DateField) sf.getType();
final String startS
= required.getFieldParam(f,FacetParams.FACET_DATE_START);
final Date start;
@@ -600,7 +596,9 @@
while (low.before(end)) {
dmp.setNow(low);
final String lowI = ft.toInternal(low);
- final String label = ft.indexedToReadable(lowI);
+ // final String label = ft.indexedToReadable(lowI);
+ String label = ft.toExternal(low);
+
Date high = dmp.parseMath(gap);
if (end.before(high)) {
if (params.getFieldBool(f,FacetParams.FACET_DATE_HARD_END,false)) {
@@ -615,7 +613,8 @@
"date facet infinite loop (is gap negative?)");
}
final String highI = ft.toInternal(high);
- resInner.add(label, rangeCount(f,lowI,highI,true,true));
+ // resInner.add(label, rangeCount(sf,lowI,highI,true,true));
+ resInner.add(label, rangeCount(sf,low,high,true,true));
low = high;
}
} catch (java.text.ParseException e) {
@@ -647,15 +646,15 @@
if (all || others.contains(FacetDateOther.BEFORE)) {
resInner.add(FacetDateOther.BEFORE.toString(),
- rangeCount(f,null,startI,false,false));
+ rangeCount(sf,null,start,false,false));
}
if (all || others.contains(FacetDateOther.AFTER)) {
resInner.add(FacetDateOther.AFTER.toString(),
- rangeCount(f,endI,null,false,false));
+ rangeCount(sf,end,null,false,false));
}
if (all || others.contains(FacetDateOther.BETWEEN)) {
resInner.add(FacetDateOther.BETWEEN.toString(),
- rangeCount(f,startI,endI,true,true));
+ rangeCount(sf,start,end,true,true));
}
}
}
@@ -665,14 +664,20 @@
}
/**
- * Macro for getting the numDocs of a TermRangeQuery over docs
+ * Macro for getting the numDocs of range over docs
* @see SolrIndexSearcher#numDocs
* @see TermRangeQuery
*/
- protected int rangeCount(String field, String low, String high,
+ protected int rangeCount(SchemaField sf, String low, String high,
+ boolean iLow, boolean iHigh) throws IOException {
+ Query rangeQ = sf.getType().getRangeQuery(null, sf,low,high,iLow,iHigh);
+ return searcher.numDocs(rangeQ ,base);
+ }
+
+ protected int rangeCount(SchemaField sf, Date low, Date high,
boolean iLow, boolean iHigh) throws IOException {
- return searcher.numDocs(new TermRangeQuery(field,low,high,iLow,iHigh),
- base);
+ Query rangeQ = ((DateField)(sf.getType())).getRangeQuery(null, sf,low,high,iLow,iHigh);
+ return searcher.numDocs(rangeQ ,base);
}
/**
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java Mon Aug 3 21:09:05 2009
@@ -22,6 +22,8 @@
import org.apache.solr.request.TextResponseWriter;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.search.function.*;
import org.apache.solr.search.QParser;
@@ -158,7 +160,8 @@
if (0 < zz) {
math = val.substring(zz+1);
try {
- p.setNow(toObject(val.substring(0,zz)));
+ // p.setNow(toObject(val.substring(0,zz)));
+ p.setNow(parseDate(val.substring(0,zz+1)));
} catch (ParseException e) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
"Invalid Date in Date Math String:'"
@@ -193,6 +196,7 @@
public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
+
public Date toObject(String indexedForm) throws java.text.ParseException {
return parseDate(indexedToReadable(indexedForm));
}
@@ -244,6 +248,13 @@
}
/**
+ * Return the standard human readable form of the date
+ */
+ public String toExternal(Date d) {
+ return fmtThreadLocal.get().format(d) + 'Z';
+ }
+
+ /**
* Thread safe method that can be used by subclasses to parse a Date
* that is already in the internal representation
*/
@@ -336,6 +347,16 @@
public ValueSource getValueSource(SchemaField field, QParser parser) {
return new DateFieldSource(field.getName(), field.getType());
}
+
+ /** DateField specific range query */
+ public Query getRangeQuery(QParser parser, SchemaField sf, Date part1, Date part2, boolean minInclusive, boolean maxInclusive) {
+ return new TermRangeQuery(
+ sf.getName(),
+ part1 == null ? null : toInternal(part1),
+ part2 == null ? null : toInternal(part2),
+ minInclusive, maxInclusive);
+ }
+
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java Mon Aug 3 21:09:05 2009
@@ -233,7 +233,7 @@
/**
* Convert an external value (from XML update command or from query string)
- * into the internal format.
+ * into the internal format for both storing and indexing (which can be modified by any analyzers).
* @see #toExternal
*/
public String toInternal(String val) {
@@ -280,7 +280,11 @@
// that the indexed form is the same as the stored field form.
return f.stringValue();
}
-
+
+ /** Given the readable value, return the term value that will match it. */
+ public String readableToIndexed(String val) {
+ return toInternal(val);
+ }
/*********
// default analyzer for non-text fields.
@@ -437,7 +441,7 @@
* handle nulls in part1 and/or part2 as well as unequal minInclusive and maxInclusive parameters gracefully.
*
* @param parser
- * @param field the name of the field
+ * @param field the schema field
* @param part1 the lower boundary of the range, nulls are allowed.
* @param part2 the upper boundary of the range, nulls are allowed
* @param minInclusive whether the minimum of the range is inclusive or not
@@ -446,10 +450,10 @@
*
* @see org.apache.solr.search.SolrQueryParser#getRangeQuery(String, String, String, boolean)
*/
- public Query getRangeQuery(QParser parser, String field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
+ public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
// constant score mode is now enabled per default
return new TermRangeQuery(
- field,
+ field.getName(),
part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2),
minInclusive, maxInclusive);
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java Mon Aug 3 21:09:05 2009
@@ -17,10 +17,14 @@
package org.apache.solr.schema;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.solr.analysis.*;
import org.apache.solr.common.SolrException;
import org.apache.solr.request.TextResponseWriter;
@@ -30,6 +34,7 @@
import java.io.IOException;
import java.util.Map;
+import java.util.Date;
/**
* Provides field types to support for Lucene's Trie Range Queries.
@@ -43,7 +48,7 @@
* Trie fields are sortable in numerical order and can be used in function queries.
* <p/>
* Note that if you use a precisionStep of 32 for int/float and 64 for long/double, then multiple terms will not be
- * generated, range search will be no faster than any other number field, but sorting will be possible.
+ * generated, range search will be no faster than any other number field, but sorting will still be possible.
*
* @version $Id$
* @see org.apache.lucene.search.NumericRangeQuery
@@ -52,25 +57,27 @@
public class TrieField extends FieldType {
public static final int DEFAULT_PRECISION_STEP = 8;
- protected int precisionStep = TrieField.DEFAULT_PRECISION_STEP;
+ protected int precisionStepArg = TrieField.DEFAULT_PRECISION_STEP; // the one passed in or defaulted
+ protected int precisionStep; // normalized
protected TrieTypes type;
/**
* Used for handling date types following the same semantics as DateField
*/
- private static final DateField dateField = new DateField();
+ static final DateField dateField = new DateField();
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
String p = args.remove("precisionStep");
if (p != null) {
- precisionStep = Integer.parseInt(p);
+ precisionStepArg = Integer.parseInt(p);
}
+ // normalize the precisionStep
+ precisionStep = precisionStepArg;
+ if (precisionStep<=0 || precisionStep>=64) precisionStep=Integer.MAX_VALUE;
String t = args.remove("type");
- if (t == null) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Invalid type specified in schema.xml for field: " + args.get("name"));
- } else {
+
+ if (t != null) {
try {
type = TrieTypes.valueOf(t.toUpperCase());
} catch (IllegalArgumentException e) {
@@ -78,6 +85,7 @@
"Invalid type specified in schema.xml for field: " + args.get("name"), e);
}
}
+
CharFilterFactory[] filterFactories = new CharFilterFactory[0];
TokenFilterFactory[] tokenFilterFactories = new TokenFilterFactory[0];
@@ -88,18 +96,19 @@
@Override
public Object toObject(Fieldable f) {
- String s = f.stringValue();
+ byte[] arr = f.binaryValue();
+ if (arr==null) return badFieldString(f);
switch (type) {
case INTEGER:
- return Integer.parseInt(s);
+ return toInt(arr);
case FLOAT:
- return Float.parseFloat(s);
+ return toFloat(arr);
case LONG:
- return Long.parseLong(s);
+ return toLong(arr);
case DOUBLE:
- return Double.parseDouble(s);
+ return toDouble(arr);
case DATE:
- return dateField.toObject(f);
+ return new Date(toLong(arr));
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
@@ -111,7 +120,7 @@
return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_INT_PARSER, top);
case FLOAT:
return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER, top);
- case DATE:
+ case DATE: // fallthrough
case LONG:
return new SortField(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, top);
case DOUBLE:
@@ -128,6 +137,7 @@
case FLOAT:
return new FloatFieldSource(field.getName(), FieldCache.NUMERIC_UTILS_FLOAT_PARSER);
case DATE:
+ return new TrieDateFieldSource(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER);
case LONG:
return new LongFieldSource(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER);
case DOUBLE:
@@ -138,11 +148,58 @@
}
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
- xmlWriter.writeVal(name, toObject(f));
+ byte[] arr = f.binaryValue();
+ if (arr==null) {
+ xmlWriter.writeStr(name, badFieldString(f));
+ return;
+ }
+
+ switch (type) {
+ case INTEGER:
+ xmlWriter.writeInt(name,toInt(arr));
+ break;
+ case FLOAT:
+ xmlWriter.writeFloat(name,toFloat(arr));
+ break;
+ case LONG:
+ xmlWriter.writeLong(name,toLong(arr));
+ break;
+ case DOUBLE:
+ xmlWriter.writeDouble(name,toDouble(arr));
+ break;
+ case DATE:
+ xmlWriter.writeDate(name,new Date(toLong(arr)));
+ break;
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
+ }
}
public void write(TextResponseWriter writer, String name, Fieldable f) throws IOException {
- writer.writeVal(name, toObject(f));
+ byte[] arr = f.binaryValue();
+ if (arr==null) {
+ writer.writeStr(name, badFieldString(f),true);
+ return;
+ }
+ switch (type) {
+ case INTEGER:
+ writer.writeInt(name,toInt(arr));
+ break;
+ case FLOAT:
+ writer.writeFloat(name,toFloat(arr));
+ break;
+ case LONG:
+ writer.writeLong(name,toLong(arr));
+ break;
+ case DOUBLE:
+ writer.writeDouble(name,toDouble(arr));
+ break;
+ case DATE:
+ writer.writeDate(name,new Date(toLong(arr)));
+ break;
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
+ }
}
@Override
@@ -154,7 +211,7 @@
* @return the precisionStep used to index values into the field
*/
public int getPrecisionStep() {
- return precisionStep;
+ return precisionStepArg;
}
/**
@@ -165,35 +222,38 @@
}
@Override
- public Query getRangeQuery(QParser parser, String field, String min, String max, boolean minInclusive, boolean maxInclusive) {
+ public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
+ // don't use a precisionStep if the field is multiValued
+ int ps = field.multiValued() ? Integer.MAX_VALUE : precisionStep;
+
Query query = null;
switch (type) {
case INTEGER:
- query = NumericRangeQuery.newIntRange(field, precisionStep,
+ query = NumericRangeQuery.newIntRange(field.getName(), ps,
min == null ? null : Integer.parseInt(min),
max == null ? null : Integer.parseInt(max),
minInclusive, maxInclusive);
break;
case FLOAT:
- query = NumericRangeQuery.newFloatRange(field, precisionStep,
+ query = NumericRangeQuery.newFloatRange(field.getName(), ps,
min == null ? null : Float.parseFloat(min),
max == null ? null : Float.parseFloat(max),
minInclusive, maxInclusive);
break;
case LONG:
- query = NumericRangeQuery.newLongRange(field, precisionStep,
+ query = NumericRangeQuery.newLongRange(field.getName(), ps,
min == null ? null : Long.parseLong(min),
max == null ? null : Long.parseLong(max),
minInclusive, maxInclusive);
break;
case DOUBLE:
- query = NumericRangeQuery.newDoubleRange(field, precisionStep,
+ query = NumericRangeQuery.newDoubleRange(field.getName(), ps,
min == null ? null : Double.parseDouble(min),
max == null ? null : Double.parseDouble(max),
minInclusive, maxInclusive);
break;
case DATE:
- query = NumericRangeQuery.newLongRange(field, precisionStep,
+ query = NumericRangeQuery.newLongRange(field.getName(), ps,
min == null ? null : dateField.parseMath(null, min).getTime(),
max == null ? null : dateField.parseMath(null, max).getTime(),
minInclusive, maxInclusive);
@@ -205,24 +265,196 @@
return query;
}
+
+ static int toInt(byte[] arr) {
+ return (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
+ }
+
+ static long toLong(byte[] arr) {
+ int high = (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
+ int low = (arr[4]<<24) | ((arr[5]&0xff)<<16) | ((arr[6]&0xff)<<8) | (arr[7]&0xff);
+ return (((long)high)<<32) | (low&0x0ffffffffL);
+ }
+
+ static float toFloat(byte[] arr) {
+ return Float.intBitsToFloat(toInt(arr));
+ }
+
+ static double toDouble(byte[] arr) {
+ return Double.longBitsToDouble(toLong(arr));
+ }
+
+ static byte[] toArr(int val) {
+ byte[] arr = new byte[4];
+ arr[0] = (byte)(val>>>24);
+ arr[1] = (byte)(val>>>16);
+ arr[2] = (byte)(val>>>8);
+ arr[3] = (byte)(val);
+ return arr;
+ }
+
+ static byte[] toArr(long val) {
+ byte[] arr = new byte[8];
+ arr[0] = (byte)(val>>>56);
+ arr[1] = (byte)(val>>>48);
+ arr[2] = (byte)(val>>>40);
+ arr[3] = (byte)(val>>>32);
+ arr[4] = (byte)(val>>>24);
+ arr[5] = (byte)(val>>>16);
+ arr[6] = (byte)(val>>>8);
+ arr[7] = (byte)(val);
+ return arr;
+ }
+
+ static byte[] toArr(float val) {
+ return toArr(Float.floatToRawIntBits(val));
+ }
+
+ static byte[] toArr(double val) {
+ return toArr(Double.doubleToRawLongBits(val));
+ }
+
+
+ @Override
+ public String storedToReadable(Fieldable f) {
+ return toExternal(f);
+ }
+
+ @Override
+ public String readableToIndexed(String val) {
+ switch (type) {
+ case INTEGER:
+ return NumericUtils.intToPrefixCoded(Integer.parseInt(val));
+ case FLOAT:
+ return NumericUtils.intToPrefixCoded(NumericUtils.floatToSortableInt(Float.parseFloat(val)));
+ case LONG:
+ return NumericUtils.longToPrefixCoded(Long.parseLong(val));
+ case DOUBLE:
+ return NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(Double.parseDouble(val)));
+ case DATE:
+ return NumericUtils.longToPrefixCoded(dateField.parseMath(null, val).getTime());
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
+ }
+ }
+
+
@Override
public String toInternal(String val) {
- return super.toInternal(val);
+ return readableToIndexed(val);
+ }
+
+
+ static String badFieldString(Fieldable f) {
+ String s = f.stringValue();
+ return "ERROR:SCHEMA-INDEX-MISMATCH,stringValue="+s;
}
@Override
public String toExternal(Fieldable f) {
- return super.toExternal(f);
+ byte[] arr = f.binaryValue();
+ if (arr==null) return badFieldString(f);
+ switch (type) {
+ case INTEGER:
+ return Integer.toString(toInt(arr));
+ case FLOAT:
+ return Float.toString(toFloat(arr));
+ case LONG:
+ return Long.toString(toLong(arr));
+ case DOUBLE:
+ return Double.toString(toDouble(arr));
+ case DATE:
+ return dateField.formatDate(new Date(toLong(arr)));
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
+ }
}
@Override
public String indexedToReadable(String indexedForm) {
- return super.indexedToReadable(indexedForm);
+ switch (type) {
+ case INTEGER:
+ return Integer.toString( NumericUtils.prefixCodedToInt(indexedForm) );
+ case FLOAT:
+ return Float.toString( NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(indexedForm)) );
+ case LONG:
+ return Long.toString( NumericUtils.prefixCodedToLong(indexedForm) );
+ case DOUBLE:
+ return Double.toString( NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(indexedForm)) );
+ case DATE:
+ return dateField.formatDate( new Date(NumericUtils.prefixCodedToLong(indexedForm)) );
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
+ }
}
@Override
public String storedToIndexed(Fieldable f) {
- return super.storedToIndexed(f);
+ // TODO: optimize to remove redundant string conversion
+ return readableToIndexed(storedToReadable(f));
+ }
+
+ @Override
+ public Field createField(SchemaField field, String externalVal, float boost) {
+ boolean indexed = field.indexed();
+ boolean stored = field.stored();
+
+ if (!indexed && !stored) {
+ if (log.isTraceEnabled())
+ log.trace("Ignoring unindexed/unstored field: " + field);
+ return null;
+ }
+
+ int ps = field.multiValued() ? Integer.MAX_VALUE : precisionStep;
+
+ byte[] arr=null;
+ TokenStream ts=null;
+ // String indexedVal = indexed && precisionStep==0 ? readableToIndexed(externalVal) : null;
+
+ switch (type) {
+ case INTEGER:
+ int i = Integer.parseInt(externalVal);
+ if (stored) arr = toArr(i);
+ if (indexed) ts = new NumericTokenStream(ps).setIntValue(i);
+ break;
+ case FLOAT:
+ float f = Float.parseFloat(externalVal);
+ if (stored) arr = toArr(f);
+ if (indexed) ts = new NumericTokenStream(ps).setFloatValue(f);
+ break;
+ case LONG:
+ long l = Long.parseLong(externalVal);
+ if (stored) arr = toArr(l);
+ if (indexed) ts = new NumericTokenStream(ps).setLongValue(l);
+ break;
+ case DOUBLE:
+ double d = Double.parseDouble(externalVal);
+ if (stored) arr = toArr(d);
+ if (indexed) ts = new NumericTokenStream(ps).setDoubleValue(d);
+ break;
+ case DATE:
+ long time = dateField.parseMath(null, externalVal).getTime();
+ if (stored) arr = toArr(time);
+ if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
+ break;
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
+ }
+
+ Field f;
+ if (stored) {
+ f = new Field(field.getName(), arr, Field.Store.YES);
+ if (indexed) f.setTokenStream(ts);
+ } else {
+ f = new Field(field.getName(), ts);
+ }
+
+ // term vectors aren't supported
+
+ f.setOmitNorms(field.omitNorms());
+ f.setOmitTermFreqAndPositions(field.omitTf());
+ f.setBoost(boost);
+ return f;
}
public enum TrieTypes {
@@ -233,3 +465,23 @@
DATE
}
}
+
+class TrieDateFieldSource extends LongFieldSource {
+
+ public TrieDateFieldSource(String field, FieldCache.LongParser parser) {
+ super(field, parser);
+ }
+
+ public TrieDateFieldSource(String field) {
+ super(field);
+ }
+
+ public String description() {
+ return "date(" + field + ')';
+ }
+
+ @Override
+ public long externalToLong(String extVal) {
+ return TrieField.dateField.parseMath(null, extVal).getTime();
+ }
+}
\ No newline at end of file
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java Mon Aug 3 21:09:05 2009
@@ -303,7 +303,13 @@
static void writeFieldVal(String val, FieldType ft, Appendable out, int flags) throws IOException {
if (ft!=null) {
- out.append(ft.toExternal(new Field("",val, Field.Store.YES, Field.Index.UN_TOKENIZED)));
+ try {
+ out.append(ft.indexedToReadable(val));
+ } catch (Exception e) {
+ out.append("EXCEPTION(val=");
+ out.append(val);
+ out.append(")");
+ }
} else {
out.append(val);
}
@@ -349,7 +355,7 @@
if (lt==null) {
out.append('*');
} else {
- writeFieldVal(lt.toString(), ft, out, flags);
+ out.append(lt.toString());
}
out.append(" TO ");
@@ -357,7 +363,7 @@
if (ut==null) {
out.append('*');
} else {
- writeFieldVal(ut.toString(), ft, out, flags);
+ out.append(ut.toString());
}
out.append( q.includesMax() ? ']' : '}' );
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/SolrQueryParser.java Mon Aug 3 21:09:05 2009
@@ -26,6 +26,7 @@
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.TrieField;
+import org.apache.solr.schema.SchemaField;
// TODO: implement the analysis of simple fields with
// FieldType.toInternal() instead of going through the
@@ -118,8 +119,8 @@
protected Query getRangeQuery(String field, String part1, String part2, boolean inclusive) throws ParseException {
checkNullField(field);
- FieldType ft = schema.getFieldType(field);
- return ft.getRangeQuery(parser, field,
+ SchemaField sf = schema.getField(field);
+ return sf.getType().getRangeQuery(parser, sf,
"*".equals(part1) ? null : part1,
"*".equals(part2) ? null : part2,
inclusive, inclusive);
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/function/LongFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/function/LongFieldSource.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/function/LongFieldSource.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/function/LongFieldSource.java Mon Aug 3 21:09:05 2009
@@ -47,6 +47,11 @@
return "long(" + field + ')';
}
+
+ public long externalToLong(String extVal) {
+ return Long.parseLong(extVal);
+ }
+
public DocValues getValues(IndexReader reader) throws IOException {
final long[] arr = (parser == null) ?
((FieldCache) cache).getLongs(reader, field) :
@@ -76,7 +81,6 @@
return description() + '=' + longVal(doc);
}
-
@Override
public ValueSourceScorer getRangeScorer(IndexReader reader, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
long lower,upper;
@@ -86,14 +90,14 @@
if (lowerVal==null) {
lower = Long.MIN_VALUE;
} else {
- lower = Long.parseLong(lowerVal);
+ lower = externalToLong(lowerVal);
if (!includeLower && lower < Long.MAX_VALUE) lower++;
}
if (upperVal==null) {
upper = Long.MAX_VALUE;
} else {
- upper = Long.parseLong(upperVal);
+ upper = externalToLong(upperVal);
if (!includeUpper && upper > Long.MIN_VALUE) upper--;
}
@@ -116,7 +120,7 @@
}
public boolean equals(Object o) {
- if (o.getClass() != LongFieldSource.class) return false;
+ if (o.getClass() != this.getClass()) return false;
LongFieldSource other = (LongFieldSource) o;
return super.equals(other)
&& this.parser == null ? other.parser == null :
@@ -124,7 +128,7 @@
}
public int hashCode() {
- int h = parser == null ? Long.class.hashCode() : parser.getClass().hashCode();
+ int h = parser == null ? this.getClass().hashCode() : parser.getClass().hashCode();
h += super.hashCode();
return h;
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/util/AbstractSolrTestCase.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/util/AbstractSolrTestCase.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/util/AbstractSolrTestCase.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/util/AbstractSolrTestCase.java Mon Aug 3 21:09:05 2009
@@ -180,7 +180,8 @@
String results = h.validateXPath(response, tests);
if (null != results) {
fail(m + "query failed XPath: " + results +
- " xml response was: " + response);
+ "\n xml response was: " + response +
+ "\n request was: " + req.getParamString());
}
} catch (XPathExpressionException e1) {
throw new RuntimeException("XPath is invalid", e1);
Modified: lucene/solr/trunk/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/client/solrj/SolrExampleTests.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/client/solrj/SolrExampleTests.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/client/solrj/SolrExampleTests.java Mon Aug 3 21:09:05 2009
@@ -48,7 +48,7 @@
* @version $Id$
* @since solr 1.3
*/
-abstract public class SolrExampleTests extends SolrExampleTestBase
+abstract public class SolrExampleTests extends SolrExampleTestBase
{
/**
* query the example
@@ -336,6 +336,8 @@
server.deleteByQuery( "*:*" );// delete everything!
server.commit();
assertNumFound( "*:*", 0 ); // make sure it got in
+
+ String f = "val_pi";
int i=0; // 0 1 2 3 4 5 6 7 8 9
int[] nums = new int[] { 23, 26, 38, 46, 55, 63, 77, 84, 92, 94 };
@@ -343,7 +345,7 @@
SolrInputDocument doc = new SolrInputDocument();
doc.setField( "id", "doc"+i++ );
doc.setField( "name", "doc: "+num );
- doc.setField( "popularity", num );
+ doc.setField( f, num );
server.add( doc );
}
server.commit();
@@ -351,10 +353,10 @@
SolrQuery query = new SolrQuery( "*:*" );
query.setRows( 0 );
- query.setGetFieldStatistics( "popularity" );
+ query.setGetFieldStatistics( f );
QueryResponse rsp = server.query( query );
- FieldStatsInfo stats = rsp.getFieldStatsInfo().get( "popularity" );
+ FieldStatsInfo stats = rsp.getFieldStatsInfo().get( f );
assertNotNull( stats );
assertEquals( 23.0, stats.getMin() );
@@ -373,14 +375,14 @@
SolrInputDocument doc = new SolrInputDocument();
doc.setField( "id", "doc"+i++ );
doc.setField( "name", "doc: "+num );
- doc.setField( "popularity", num );
+ doc.setField( f, num );
server.add( doc );
}
server.commit();
assertNumFound( "*:*", nums.length ); // make sure they all got in
rsp = server.query( query );
- stats = rsp.getFieldStatsInfo().get( "popularity" );
+ stats = rsp.getFieldStatsInfo().get( f );
assertNotNull( stats );
assertEquals( 5.0, stats.getMin() );
@@ -399,7 +401,7 @@
SolrInputDocument doc = new SolrInputDocument();
doc.setField( "id", "doc"+i );
doc.setField( "name", "doc: "+num );
- doc.setField( "popularity", num );
+ doc.setField( f, num );
doc.setField( "inStock", i < 5 );
server.add( doc );
}
@@ -408,9 +410,9 @@
assertNumFound( "inStock:false", 5 ); // make sure they all got in
// facet on 'inStock'
- query.addStatsFieldFacets( "popularity", "inStock" );
+ query.addStatsFieldFacets( f, "inStock" );
rsp = server.query( query );
- stats = rsp.getFieldStatsInfo().get( "popularity" );
+ stats = rsp.getFieldStatsInfo().get( f );
assertNotNull( stats );
List<FieldStatsInfo> facets = stats.getFacets().get( "inStock" );
Modified: lucene/solr/trunk/src/test/org/apache/solr/search/TestQueryTypes.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/search/TestQueryTypes.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/search/TestQueryTypes.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/search/TestQueryTypes.java Mon Aug 3 21:09:05 2009
@@ -46,8 +46,65 @@
assertU(adoc("id","6", "v_f","8983"));
assertU(adoc("id","7", "v_f","1.5"));
assertU(adoc("id","8", "v_ti","5"));
+
+ Object[] arr = new Object[] {
+ "id",999.0
+ ,"v_s","wow dude"
+ ,"v_t","wow"
+ ,"v_ti",-1
+ ,"v_tis",-1
+ ,"v_tl",-1234567891234567890L
+ ,"v_tls",-1234567891234567890L
+ ,"v_tf",-2.0f
+ ,"v_tfs",-2.0f
+ ,"v_td",-2.0
+ ,"v_tds",-2.0
+ ,"v_tdt","2000-05-10T01:01:01Z"
+ ,"v_tdts","2002-08-26T01:01:01Z"
+ };
+ String[] sarr = new String[arr.length];
+ for (int i=0; i<arr.length; i++) {
+ sarr[i] = arr[i].toString();
+ }
+
+ assertU(adoc(sarr));
assertU(optimize());
+ // test field queries
+ for (int i=0; i<arr.length; i+=2) {
+ String f = arr[i].toString();
+ String v = arr[i+1].toString();
+
+ // normal lucene fielded query
+ assertQ(req( "q",f+":\""+v+'"')
+ ,"//result[@numFound='1']"
+ ,"//*[@name='id'][.='999.0']"
+ ,"//*[@name='" + f + "'][.='" + v + "']"
+ );
+ // System.out.println("#########################################" + f + "=" + v);
+
+ // field qparser
+ assertQ(req( "q", "{!field f="+f+"}"+v)
+ ,"//result[@numFound='1']"
+ );
+
+ // lucene range
+ assertQ(req( "q", f + ":[\"" + v + "\" TO \"" + v + "\"]" )
+ ,"//result[@numFound='1']"
+ );
+
+ // frange qparser
+ assertQ(req( "q", "{!frange v="+f+" l='"+v+"' u='"+v+"'}" )
+ ,"//result[@numFound='1']"
+ );
+
+ // function query... just make sure it doesn't throw an exception
+ assertQ(req( "q", "+id:999 _val_:\"" + f + "\"")
+ ,"//result[@numFound='1']"
+ );
+
+ }
+
// Some basic tests to ensure that parsing local params is working
assertQ("test prefix query",
@@ -85,12 +142,10 @@
,"//result[@numFound='1']"
);
- /** future test
assertQ(
req("q","{!field f=v_ti}5")
,"//result[@numFound='1']"
);
- **/
assertQ("test multi term field query on text type",
req("q","{!field f=v_t}Hello DUDE")
Modified: lucene/solr/trunk/src/test/org/apache/solr/search/TestRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/search/TestRangeQuery.java?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/search/TestRangeQuery.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/search/TestRangeQuery.java Mon Aug 3 21:09:05 2009
@@ -94,6 +94,7 @@
norm_fields.put("foo_ti", ints);
norm_fields.put("foo_tl", longs);
norm_fields.put("foo_td", doubles);
+ norm_fields.put("foo_tdt", dates);
norm_fields.put("foo_s", strings);
norm_fields.put("foo_dt", dates);
@@ -108,6 +109,7 @@
frange_fields.put("foo_ti", ints);
frange_fields.put("foo_tl", longs);
frange_fields.put("foo_td", doubles);
+ frange_fields.put("foo_tdt", dates);
frange_fields.put("foo_pi", ints);
frange_fields.put("foo_pl", longs);
Modified: lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml (original)
+++ lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml Mon Aug 3 21:09:05 2009
@@ -88,14 +88,14 @@
<fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
- <fieldType name="tint" class="solr.TrieField" type="integer" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tfloat" class="solr.TrieField" type="float" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tlong" class="solr.TrieField" type="long" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tdouble" class="solr.TrieField" type="double" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <fieldType name="tint" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <fieldType name="tfloat" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <fieldType name="tlong" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <fieldType name="tdouble" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tdouble4" class="solr.TrieField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <fieldType name="tdouble4" class="solr.TrieDoubleField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tdate" class="solr.TrieField" type="date" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
Modified: lucene/solr/trunk/src/test/test-files/solr/conf/schema11.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/test-files/solr/conf/schema11.xml?rev=800565&r1=800564&r2=800565&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/test-files/solr/conf/schema11.xml (original)
+++ lucene/solr/trunk/src/test/test-files/solr/conf/schema11.xml Mon Aug 3 21:09:05 2009
@@ -237,15 +237,19 @@
<fieldType name="file" keyField="id" defVal="1" stored="false" indexed="false" class="solr.ExternalFileField" valType="float"/>
- <fieldType name="tint" class="solr.TrieField" type="integer" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tfloat" class="solr.TrieField" type="float" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tlong" class="solr.TrieField" type="long" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
- <fieldType name="tdouble" class="solr.TrieField" type="double" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <fieldType name="tint" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble4" class="solr.TrieDoubleField" precisionStep="4" omitNorms="true" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0"/>
- <fieldType name="tdouble4" class="solr.TrieField" type="double" precisionStep="4" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
-
- <fieldType name="tdate" class="solr.TrieField" type="date" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
+ <fieldType name="tints" class="solr.TrieIntField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
+ <fieldType name="tfloats" class="solr.TrieFloatField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
+ <fieldType name="tdates" class="solr.TrieDateField" omitNorms="true" positionIncrementGap="0" precisionStep="0" multiValued="true" />
</types>
@@ -298,6 +302,12 @@
<dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/>
<dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/>
+ <dynamicField name="*_tis" type="tints" indexed="true" stored="true"/>
+ <dynamicField name="*_tls" type="tlongs" indexed="true" stored="true"/>
+ <dynamicField name="*_tfs" type="tfloats" indexed="true" stored="true"/>
+ <dynamicField name="*_tds" type="tdoubles" indexed="true" stored="true"/>
+ <dynamicField name="*_tdts" type="tdates" indexed="true" stored="true"/>
+
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>