You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2009/04/12 13:03:10 UTC
svn commit: r764291 - in /lucene/solr/trunk: ./ example/solr/conf/ lib/
src/java/org/apache/solr/analysis/
src/java/org/apache/solr/handler/component/
src/java/org/apache/solr/highlight/ src/java/org/apache/solr/schema/
src/test/org/apache/solr/ src/te...
Author: shalin
Date: Sun Apr 12 11:03:09 2009
New Revision: 764291
URL: http://svn.apache.org/viewvc?rev=764291&view=rev
Log:
SOLR-940 and SOLR-1079 -- Updating Lucene jars, updating Trie field types per Lucene's changes, adding ReverseStringFilterFactory, updates related to LUCENE-1500.
Committing all the above changes in one go to avoid compile errors due to Lucene API updates (except for ReverseStringFilterFactory).
Added:
lucene/solr/trunk/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java (with props)
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/example/solr/conf/schema.xml
lucene/solr/trunk/lib/lucene-analyzers-2.9-dev.jar
lucene/solr/trunk/lib/lucene-core-2.9-dev.jar
lucene/solr/trunk/lib/lucene-highlighter-2.9-dev.jar
lucene/solr/trunk/lib/lucene-memory-2.9-dev.jar
lucene/solr/trunk/lib/lucene-queries-2.9-dev.jar
lucene/solr/trunk/lib/lucene-snowball-2.9-dev.jar
lucene/solr/trunk/lib/lucene-spellchecker-2.9-dev.jar
lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieIndexTokenizerFactory.java
lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieQueryTokenizerFactory.java
lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java
lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
lucene/solr/trunk/src/java/org/apache/solr/schema/FieldProperties.java
lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java
lucene/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java
lucene/solr/trunk/src/java/org/apache/solr/schema/TextField.java
lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java
lucene/solr/trunk/src/test/org/apache/solr/TestTrie.java
lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Sun Apr 12 11:03:09 2009
@@ -374,6 +374,10 @@
23. SOLR-952: Cleanup duplicated code in deprecated HighlightingUtils (hossman)
+24. Upgraded to Lucene 2.9-dev r764281 (shalin)
+
+25. SOLR-1079: Rename omitTf to omitTermFreqAndPositions (shalin)
+
Build
----------------------
1. SOLR-776: Added in ability to sign artifacts via Ant for releases (gsingers)
Modified: lucene/solr/trunk/example/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/example/solr/conf/schema.xml?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/example/solr/conf/schema.xml (original)
+++ lucene/solr/trunk/example/solr/conf/schema.xml Sun Apr 12 11:03:09 2009
@@ -39,7 +39,7 @@
not normally be changed by applications.
1.0: multiValued attribute did not exist, all fields are multiValued by nature
1.1: multiValued attribute introduced, false by default
- 1.2: omitTf attribute introduced, true by default -->
+ 1.2: omitTermFreqAndPositions attribute introduced, true by default -->
<types>
<!-- field type definitions. The "name" attribute is
@@ -120,10 +120,7 @@
human readable in the internal form. Range searches on such fields use the fast Trie Range Queries
which are much faster than range searches on the SortableNumberField types.
- For the fast range search to work, trie fields must be indexed. Trie fields are <b>not</b> sortable
- in numerical order. Also, they cannot be used in function queries. If one needs sorting as well as
- fast range search, one should create a copy field specifically for sorting. Same workaround is
- suggested for using trie fields in function queries as well.
+ For the fast range search to work, trie fields must be indexed.
For each number being added to this field, multiple terms are generated as per the algorithm described in
org.apache.lucene.search.trie package description. The possible number of terms depend on the precisionStep
@@ -131,8 +128,7 @@
value of precisionStep is 8.
Note that if you use a precisionStep of 32 for int/float and 64 for long/double, then multiple terms
- will not be generated, range search will be no faster than any other number field,
- but sorting will be possible.
+ will not be generated, and range search will be no faster than any other number field.
-->
<fieldType name="tint" class="solr.TrieField" type="integer" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
<fieldType name="tfloat" class="solr.TrieField" type="float" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
@@ -143,8 +139,7 @@
<!--
This date field manipulates the value into a trie encoded strings for fast range searches. They follow the
- same format and semantics as the normal DateField and support the date math syntax except that they are
- not sortable and cannot be used in function queries.
+ same format and semantics as the normal DateField and support the date math syntax.
-->
<fieldType name="tdate" class="solr.TrieField" type="date" omitNorms="true" positionIncrementGap="0" indexed="true" stored="false" />
Modified: lucene/solr/trunk/lib/lucene-analyzers-2.9-dev.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/lib/lucene-analyzers-2.9-dev.jar?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/solr/trunk/lib/lucene-core-2.9-dev.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/lib/lucene-core-2.9-dev.jar?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/solr/trunk/lib/lucene-highlighter-2.9-dev.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/lib/lucene-highlighter-2.9-dev.jar?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/solr/trunk/lib/lucene-memory-2.9-dev.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/lib/lucene-memory-2.9-dev.jar?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/solr/trunk/lib/lucene-queries-2.9-dev.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/lib/lucene-queries-2.9-dev.jar?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/solr/trunk/lib/lucene-snowball-2.9-dev.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/lib/lucene-snowball-2.9-dev.jar?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/solr/trunk/lib/lucene-spellchecker-2.9-dev.jar
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/lib/lucene-spellchecker-2.9-dev.jar?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
Binary files - no diff available.
Added: lucene/solr/trunk/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java?rev=764291&view=auto
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java (added)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java Sun Apr 12 11:03:09 2009
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.analysis;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.reverse.ReverseStringFilter;
+
+/**
+ * A FilterFactory which reverses the input.
+ *
+ * @version $Id$
+ * @since solr 1.4
+ */
+public class ReverseStringFilterFactory extends BaseTokenFilterFactory {
+ public ReverseStringFilter create(TokenStream in) {
+ return new ReverseStringFilter(in);
+ }
+}
+
Propchange: lucene/solr/trunk/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/src/java/org/apache/solr/analysis/ReverseStringFilterFactory.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Modified: lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieIndexTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieIndexTokenizerFactory.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieIndexTokenizerFactory.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieIndexTokenizerFactory.java Sun Apr 12 11:03:09 2009
@@ -20,6 +20,8 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.search.trie.TrieUtils;
+import org.apache.lucene.search.trie.IntTrieTokenStream;
+import org.apache.lucene.search.trie.LongTrieTokenStream;
import org.apache.solr.common.SolrException;
import org.apache.solr.schema.DateField;
import static org.apache.solr.schema.TrieField.TrieTypes;
@@ -39,8 +41,9 @@
* @since solr 1.4
*/
public class TrieIndexTokenizerFactory extends BaseTokenizerFactory {
- private final int precisionStep;
- private final TrieTypes type;
+ protected static final DateField dateField = new DateField();
+ protected final int precisionStep;
+ protected final TrieTypes type;
public TrieIndexTokenizerFactory(TrieTypes type, int precisionStep) {
this.type = type;
@@ -49,55 +52,27 @@
public TokenStream create(Reader input) {
try {
- return new TrieIndexTokenizer(input, type, precisionStep);
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to create TrieIndexTokenizer", e);
- }
- }
-
- /**
- * @version $Id$
- * @since solr 1.4
- */
- public static class TrieIndexTokenizer extends Tokenizer {
- private final String[] trieVals;
- private int pos = 0;
-
- protected static final DateField dateField = new DateField();
-
- public TrieIndexTokenizer(Reader reader, TrieTypes type, int precisionStep) throws IOException {
- super(reader);
StringBuilder builder = new StringBuilder();
char[] buf = new char[8];
int len;
- while ((len = reader.read(buf)) != -1)
+ while ((len = input.read(buf)) != -1)
builder.append(buf, 0, len);
switch (type) {
case INTEGER:
- this.trieVals = TrieUtils.trieCodeInt(Integer.parseInt(builder.toString()), precisionStep);
- break;
+ return new IntTrieTokenStream(Integer.parseInt(builder.toString()), precisionStep);
case FLOAT:
- this.trieVals = TrieUtils.trieCodeInt(TrieUtils.floatToSortableInt(Float.parseFloat(builder.toString())), precisionStep);
- break;
+ return new IntTrieTokenStream(TrieUtils.floatToSortableInt(Float.parseFloat(builder.toString())), precisionStep);
case LONG:
- this.trieVals = TrieUtils.trieCodeLong(Long.parseLong(builder.toString()), precisionStep);
- break;
+ return new LongTrieTokenStream(Long.parseLong(builder.toString()), precisionStep);
case DOUBLE:
- this.trieVals = TrieUtils.trieCodeLong(TrieUtils.doubleToSortableLong(Double.parseDouble(builder.toString())), precisionStep);
- break;
+ return new LongTrieTokenStream(TrieUtils.doubleToSortableLong(Double.parseDouble(builder.toString())), precisionStep);
case DATE:
- this.trieVals = TrieUtils.trieCodeLong(dateField.parseMath(null, builder.toString()).getTime(), precisionStep);
- break;
+ return new LongTrieTokenStream(dateField.parseMath(null, builder.toString()).getTime(), precisionStep);
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field");
}
- }
-
- public Token next(Token token) {
- if (pos >= trieVals.length) return null;
- token.reinit(trieVals[pos++], 0, 0);
- token.setPositionIncrement(0);
- return token;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to create TrieIndexTokenizer", e);
}
}
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieQueryTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieQueryTokenizerFactory.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieQueryTokenizerFactory.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/analysis/TrieQueryTokenizerFactory.java Sun Apr 12 11:03:09 2009
@@ -16,9 +16,8 @@
*/
package org.apache.solr.analysis;
-import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.KeywordTokenizer;
import org.apache.lucene.search.trie.TrieUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.schema.DateField;
@@ -26,6 +25,7 @@
import java.io.IOException;
import java.io.Reader;
+import java.io.StringReader;
/**
* Query time tokenizer for trie fields. It uses methods in TrieUtils to create a prefix coded representation of the
@@ -39,7 +39,8 @@
* @since solr 1.4
*/
public class TrieQueryTokenizerFactory extends BaseTokenizerFactory {
- private final TrieField.TrieTypes type;
+ protected static final DateField dateField = new DateField();
+ protected final TrieField.TrieTypes type;
public TrieQueryTokenizerFactory(TrieField.TrieTypes type) {
this.type = type;
@@ -47,34 +48,12 @@
public TokenStream create(Reader reader) {
try {
- return new TrieQueryTokenizer(reader, type);
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to create TrieQueryTokenizer", e);
- }
- }
-
- public static class TrieQueryTokenizer extends Tokenizer {
- private static final DateField dateField = new DateField();
-
- private final String number;
- private boolean used = false;
- private TrieField.TrieTypes type;
-
- public TrieQueryTokenizer(Reader reader, TrieField.TrieTypes type) throws IOException {
- super(reader);
- this.type = type;
StringBuilder builder = new StringBuilder();
char[] buf = new char[8];
int len;
while ((len = reader.read(buf)) != -1)
builder.append(buf, 0, len);
- number = builder.toString();
- }
-
- @Override
- public Token next(Token token) throws IOException {
- if (used) return null;
- String value = number;
+ String value, number = builder.toString();
switch (type) {
case INTEGER:
value = TrieUtils.intToPrefixCoded(Integer.parseInt(number));
@@ -94,10 +73,9 @@
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field");
}
- token.reinit(value, 0, 0);
- token.setPositionIncrement(0);
- used = true;
- return token;
+ return new KeywordTokenizer(new StringReader(value));
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to create trie query tokenizer", e);
}
}
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/component/QueryComponent.java Sun Apr 12 11:03:09 2009
@@ -17,7 +17,7 @@
package org.apache.solr.handler.component;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.*;
@@ -180,7 +180,7 @@
SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
ScoreDoc sd = new ScoreDoc(0,1.0f); // won't work for comparators that look at the score
NamedList sortVals = new NamedList(); // order is important for the sort fields
- StringFieldable field = new StringFieldable();
+ Field field = new Field("dummy", "", Field.Store.YES, Field.Index.NO); // a dummy Field
for (SortField sortField: sortFields) {
int type = sortField.getType();
@@ -229,7 +229,7 @@
// indexedToReadable() should be a no-op and should
// thus be harmless anyway (for all current ways anyway)
if (val instanceof String) {
- field.val = (String)val;
+ field.setValue((String)val);
val = ft.toObject(field);
}
vals.add(val);
@@ -728,104 +728,6 @@
};
}
- static class StringFieldable implements Fieldable {
- public String val;
-
- public void setBoost(float boost) {
- }
-
- public float getBoost() {
- return 0;
- }
-
- public String name() {
- return null;
- }
-
- public String stringValue() {
- return val;
- }
-
- public Reader readerValue() {
- return null;
- }
-
- public byte[] binaryValue() {
- return new byte[0];
- }
-
- public TokenStream tokenStreamValue() {
- return null;
- }
-
- public boolean isStored() {
- return true;
- }
-
- public boolean isIndexed() {
- return true;
- }
-
- public boolean isTokenized() {
- return true;
- }
-
- public boolean isCompressed() {
- return false;
- }
-
- public boolean isTermVectorStored() {
- return false;
- }
-
- public boolean isStoreOffsetWithTermVector() {
- return false;
- }
-
- public boolean isStorePositionWithTermVector() {
- return false;
- }
-
- public boolean isBinary() {
- return false;
- }
-
- public boolean getOmitNorms() {
- return false;
- }
-
- public void setOmitNorms(boolean omitNorms) {
- }
-
- public void setOmitTf(boolean omitTf) {
- }
-
- public boolean getOmitTf() {
- return false;
- }
-
- public boolean isLazy() {
- return false;
- }
-
- public int getBinaryOffset() {
- return 0;
- }
-
- public int getBinaryLength() {
- return 0;
- }
-
- public byte[] getBinaryValue() {
- return new byte[0];
- }
-
- public byte[] getBinaryValue(byte[] result) {
- return new byte[0];
- }
- }
-
-
/////////////////////////////////////////////
/// SolrInfoMBean
////////////////////////////////////////////
Modified: lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java Sun Apr 12 11:03:09 2009
@@ -38,13 +38,7 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.highlight.Formatter;
-import org.apache.lucene.search.highlight.Fragmenter;
-import org.apache.lucene.search.highlight.Highlighter;
-import org.apache.lucene.search.highlight.QueryScorer;
-import org.apache.lucene.search.highlight.SpanScorer;
-import org.apache.lucene.search.highlight.TextFragment;
-import org.apache.lucene.search.highlight.TokenSources;
+import org.apache.lucene.search.highlight.*;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.HighlightParams;
import org.apache.solr.common.params.SolrParams;
@@ -309,12 +303,16 @@
} else {
highlighter.setMaxDocCharsToAnalyze(maxCharsToAnalyze);
}
-
- TextFragment[] bestTextFragments = highlighter.getBestTextFragments(tstream, docTexts[j], mergeContiguousFragments, numFragments);
- for (int k = 0; k < bestTextFragments.length; k++) {
- if ((bestTextFragments[k] != null) && (bestTextFragments[k].getScore() > 0)) {
- frags.add(bestTextFragments[k]);
+
+ try {
+ TextFragment[] bestTextFragments = highlighter.getBestTextFragments(tstream, docTexts[j], mergeContiguousFragments, numFragments);
+ for (int k = 0; k < bestTextFragments.length; k++) {
+ if ((bestTextFragments[k] != null) && (bestTextFragments[k].getScore() > 0)) {
+ frags.add(bestTextFragments[k]);
+ }
}
+ } catch (InvalidTokenOffsetsException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
// sort such that the fragments with the highest score come first
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/FieldProperties.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/FieldProperties.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/FieldProperties.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/FieldProperties.java Sun Apr 12 11:03:09 2009
@@ -35,7 +35,7 @@
final static int BINARY = 0x00000008;
final static int COMPRESSED = 0x00000010;
final static int OMIT_NORMS = 0x00000020;
- final static int OMIT_TF = 0x00000040;
+ final static int OMIT_TF_POSITIONS = 0x00000040;
final static int STORE_TERMVECTORS = 0x00000080;
final static int STORE_TERMPOSITIONS = 0x00000100;
final static int STORE_TERMOFFSETS = 0x00000200;
@@ -49,7 +49,7 @@
static final String[] propertyNames = {
"indexed", "tokenized", "stored",
- "binary", "compressed", "omitNorms", "omitTf",
+ "binary", "compressed", "omitNorms", "omitTermFreqAndPositions",
"termVectors", "termPositions", "termOffsets",
"multiValued",
"sortMissingFirst","sortMissingLast","required"
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/FieldType.java Sun Apr 12 11:03:09 2009
@@ -88,10 +88,10 @@
// Handle additional arguments...
void setArgs(IndexSchema schema, Map<String,String> args) {
- // default to STORED, INDEXED, OMIT_TF and MULTIVALUED depending on schema version
+ // default to STORED, INDEXED, OMIT_TF_POSITIONS and MULTIVALUED depending on schema version
properties = (STORED | INDEXED);
if (schema.getVersion()< 1.1f) properties |= MULTIVALUED;
- if (schema.getVersion()> 1.1f) properties |= OMIT_TF;
+ if (schema.getVersion()> 1.1f) properties |= OMIT_TF_POSITIONS;
this.args=args;
Map<String,String> initArgs = new HashMap<String,String>(args);
@@ -202,7 +202,7 @@
getFieldIndex(field, val),
getFieldTermVec(field, val));
f.setOmitNorms(field.omitNorms());
- f.setOmitTf(field.omitTf());
+ f.setOmitTermFreqAndPositions(field.omitTf());
f.setBoost(boost);
return f;
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java Sun Apr 12 11:03:09 2009
@@ -79,7 +79,7 @@
public boolean storeTermPositions() { return (properties & STORE_TERMPOSITIONS)!=0; }
public boolean storeTermOffsets() { return (properties & STORE_TERMOFFSETS)!=0; }
public boolean omitNorms() { return (properties & OMIT_NORMS)!=0; }
- public boolean omitTf() { return (properties & OMIT_TF)!=0; }
+ public boolean omitTf() { return (properties & OMIT_TF_POSITIONS)!=0; }
public boolean multiValued() { return (properties & MULTIVALUED)!=0; }
public boolean sortMissingFirst() { return (properties & SORT_MISSING_FIRST)!=0; }
public boolean sortMissingLast() { return (properties & SORT_MISSING_LAST)!=0; }
@@ -137,7 +137,7 @@
}
if (on(falseProps,INDEXED)) {
- int pp = (INDEXED | OMIT_NORMS | OMIT_TF
+ int pp = (INDEXED | OMIT_NORMS | OMIT_TF_POSITIONS
| STORE_TERMVECTORS | STORE_TERMPOSITIONS | STORE_TERMOFFSETS
| SORT_MISSING_FIRST | SORT_MISSING_LAST);
if (on(pp,trueProps)) {
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/TextField.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/TextField.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/TextField.java Sun Apr 12 11:03:09 2009
@@ -32,7 +32,7 @@
public class TextField extends CompressableField {
protected void init(IndexSchema schema, Map<String,String> args) {
properties |= TOKENIZED;
- if (schema.getVersion()> 1.1f) properties &= ~OMIT_TF;
+ if (schema.getVersion()> 1.1f) properties &= ~OMIT_TF_POSITIONS;
super.init(schema, args);
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/TrieField.java Sun Apr 12 11:03:09 2009
@@ -29,6 +29,7 @@
import org.apache.solr.request.TextResponseWriter;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.search.QParser;
+import org.apache.solr.search.function.*;
import java.io.IOException;
import java.util.Map;
@@ -121,6 +122,23 @@
}
}
+ public ValueSource getValueSource(SchemaField field) {
+ switch (type) {
+ case INTEGER:
+ return new IntFieldSource(field.getName(), TrieUtils.FIELD_CACHE_INT_PARSER);
+ case FLOAT:
+ return new FloatFieldSource(field.getName(), TrieUtils.FIELD_CACHE_FLOAT_PARSER);
+ case LONG:
+ return new LongFieldSource(field.getName(), TrieUtils.FIELD_CACHE_LONG_PARSER);
+ case DOUBLE:
+ return new DoubleFieldSource(field.getName(), TrieUtils.FIELD_CACHE_DOUBLE_PARSER);
+ case DATE:
+ return new LongFieldSource(field.getName(), TrieUtils.FIELD_CACHE_LONG_PARSER);
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + field.name);
+ }
+ }
+
public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeVal(name, toObject(f));
}
@@ -153,31 +171,31 @@
Query query = null;
switch (type) {
case INTEGER:
- query = new IntTrieRangeFilter(field, field, precisionStep,
+ query = new IntTrieRangeFilter(field, precisionStep,
min == null ? null : Integer.parseInt(min),
max == null ? null : Integer.parseInt(max),
minInclusive, maxInclusive).asQuery();
break;
case FLOAT:
- query = new IntTrieRangeFilter(field, field, precisionStep,
+ query = new IntTrieRangeFilter(field, precisionStep,
min == null ? null : TrieUtils.floatToSortableInt(Float.parseFloat(min)),
max == null ? null : TrieUtils.floatToSortableInt(Float.parseFloat(max)),
minInclusive, maxInclusive).asQuery();
break;
case LONG:
- query = new LongTrieRangeFilter(field, field, precisionStep,
+ query = new LongTrieRangeFilter(field, precisionStep,
min == null ? null : Long.parseLong(min),
max == null ? null : Long.parseLong(max),
minInclusive, maxInclusive).asQuery();
break;
case DOUBLE:
- query = new LongTrieRangeFilter(field, field, precisionStep,
+ query = new LongTrieRangeFilter(field, precisionStep,
min == null ? null : TrieUtils.doubleToSortableLong(Double.parseDouble(min)),
max == null ? null : TrieUtils.doubleToSortableLong(Double.parseDouble(max)),
minInclusive, maxInclusive).asQuery();
break;
case DATE:
- query = new LongTrieRangeFilter(field, field, precisionStep,
+ query = new LongTrieRangeFilter(field, precisionStep,
min == null ? null : dateField.parseMath(null, min).getTime(),
max == null ? null : dateField.parseMath(null, max).getTime(),
minInclusive, maxInclusive).asQuery();
Modified: lucene/solr/trunk/src/test/org/apache/solr/TestTrie.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/TestTrie.java?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/TestTrie.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/TestTrie.java Sun Apr 12 11:03:09 2009
@@ -56,6 +56,13 @@
assertQ("Range filter tint:[-9 to *] must match 20 documents", req("q", "*:*", "fq", "tint:[-10 TO *]"), "//*[@numFound='20']");
assertQ("Range filter tint:[* to 9] must match 20 documents", req("q", "*:*", "fq", "tint:[* TO 10]"), "//*[@numFound='20']");
assertQ("Range filter tint:[* to *] must match 20 documents", req("q", "*:*", "fq", "tint:[* TO *]"), "//*[@numFound='20']");
+
+ // Sorting
+ assertQ("Sort descending does not work correctly on tint fields", req("q", "*:*", "sort", "tint desc"), "//*[@numFound='20']", "//int[@name='tint'][.='9']");
+ assertQ("Sort ascending does not work correctly on tint fields", req("q", "*:*", "sort", "tint asc"), "//*[@numFound='20']", "//int[@name='tint'][.='-10']");
+
+ // Function queries
+ assertQ("Function queries does not work correctly on tint fields", req("q", "_val_:\"sum(tint,1)\""), "//*[@numFound='20']", "//int[@name='tint'][.='9']");
}
public void testTrieTermQuery() throws Exception {
@@ -90,6 +97,13 @@
assertQ("Range filter must match only 5 documents", req, "//*[@numFound='9']");
req = req("q", "*:*", "fq", "tfloat:[0 TO *]");
assertQ("Range filter must match 10 documents", req, "//*[@numFound='10']");
+
+ // Sorting
+ assertQ("Sort descending does not work correctly on tfloat fields", req("q", "*:*", "sort", "tfloat desc"), "//*[@numFound='10']", "//float[@name='tfloat'][.='2519.9102']");
+ assertQ("Sort ascending does not work correctly on tfloat fields", req("q", "*:*", "sort", "tfloat asc"), "//*[@numFound='10']", "//float[@name='tfloat'][.='0.0']");
+
+ // Function queries
+ assertQ("Function queries does not work correctly on tfloat fields", req("q", "_val_:\"sum(tfloat,1.0)\""), "//*[@numFound='10']", "//float[@name='tfloat'][.='2519.9102']");
}
public void testTrieLongRangeSearch() throws Exception {
@@ -101,6 +115,13 @@
SolrQueryRequest req = req("q", "*:*", "fq", fq);
assertQ("Range filter must match only 5 documents", req, "//*[@numFound='6']");
assertQ("Range filter tlong:[* to *] must match 10 documents", req("q", "*:*", "fq", "tlong:[* TO *]"), "//*[@numFound='10']");
+
+ // Sorting
+ assertQ("Sort descending does not work correctly on tlong fields", req("q", "*:*", "sort", "tlong desc"), "//*[@numFound='10']", "//long[@name='tlong'][.='2147483656']");
+ assertQ("Sort ascending does not work correctly on tlong fields", req("q", "*:*", "sort", "tlong asc"), "//*[@numFound='10']", "//long[@name='tlong'][.='2147483647']");
+
+ // Function queries
+ assertQ("Function queries does not work correctly on tlong fields", req("q", "_val_:\"sum(tlong,1.0)\""), "//*[@numFound='10']", "//long[@name='tlong'][.='2147483656']");
}
public void testTrieDoubleRangeSearch() throws Exception {
@@ -111,6 +132,13 @@
String fq = "tdouble:[" + Integer.MAX_VALUE * 2.33d + " TO " + (5l + Integer.MAX_VALUE) * 2.33d + "]";
assertQ("Range filter must match only 5 documents", req("q", "*:*", "fq", fq), "//*[@numFound='6']");
assertQ("Range filter tdouble:[* to *] must match 10 documents", req("q", "*:*", "fq", "tdouble:[* TO *]"), "//*[@numFound='10']");
+
+ // Sorting
+ assertQ("Sort descending does not work correctly on tdouble fields", req("q", "*:*", "sort", "tdouble desc"), "//*[@numFound='10']", "//double[@name='tdouble'][.='5.0036369184800005E9']");
+ assertQ("Sort ascending does not work correctly on tdouble fields", req("q", "*:*", "sort", "tdouble asc"), "//*[@numFound='10']", "//double[@name='tdouble'][.='5.00363689751E9']");
+
+ // Function queries
+ assertQ("Function queries does not work correctly on tdouble fields", req("q", "_val_:\"sum(tdouble,1.0)\""), "//*[@numFound='10']", "//double[@name='tdouble'][.='5.0036369184800005E9']");
}
public void testTrieDateRangeSearch() throws Exception {
@@ -147,6 +175,13 @@
assertU(commit());
assertQ("Term query must match only 1 document", req("q", "tdate:1995-12-31T23\\:59\\:59.999Z"), "//*[@numFound='1']");
assertQ("Term query must match only 1 document", req("q", "*:*", "fq", "tdate:1995-12-31T23\\:59\\:59.999Z"), "//*[@numFound='1']");
+
+ // Sorting
+ assertQ("Sort descending does not work correctly on tdate fields", req("q", "*:*", "sort", "tdate desc"), "//*[@numFound='11']", "//date[@name='tdate'][.='2009-04-21T00:00:00Z']");
+ assertQ("Sort ascending does not work correctly on tdate fields", req("q", "*:*", "sort", "tdate asc"), "//*[@numFound='11']", "//date[@name='tdate'][.='2009-04-12T00:00:00Z']");
+
+ // Function queries
+ assertQ("Function queries does not work correctly on tdate fields", req("q", "_val_:\"sum(tdate,1.0)\""), "//*[@numFound='11']", "//date[@name='tdate'][.='2009-04-21T00:00:00Z']");
}
public void testTrieDoubleRangeSearch_CustomPrecisionStep() throws Exception {
Modified: lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml?rev=764291&r1=764290&r2=764291&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml (original)
+++ lucene/solr/trunk/src/test/test-files/solr/conf/schema-trie.xml Sun Apr 12 11:03:09 2009
@@ -272,10 +272,10 @@
<field name="id" type="sfloat" indexed="true" stored="true" required="true" />
<field name="text" type="text" indexed="true" stored="false" />
- <field name="tint" type="tint" indexed="true" stored="false" />
- <field name="tfloat" type="tfloat" indexed="true" stored="false" />
- <field name="tlong" type="tlong" indexed="true" stored="false" />
- <field name="tdouble" type="tdouble" indexed="true" stored="false" />
+ <field name="tint" type="tint" indexed="true" stored="true" />
+ <field name="tfloat" type="tfloat" indexed="true" stored="true" />
+ <field name="tlong" type="tlong" indexed="true" stored="true" />
+ <field name="tdouble" type="tdouble" indexed="true" stored="true" />
<field name="tdouble4" type="tdouble4" indexed="true" stored="false" />