You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ry...@apache.org on 2011/03/29 19:41:58 UTC
svn commit: r1086651 - in /lucene/dev/trunk: lucene/CHANGES.txt
lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java
solr/src/java/org/apache/solr/schema/TrieDateField.java
solr/src/java/org/apache/solr/schema/TrieField.java
Author: ryan
Date: Tue Mar 29 17:41:58 2011
New Revision: 1086651
URL: http://svn.apache.org/viewvc?rev=1086651&view=rev
Log:
LUCENE-3001 -- Adding TrieFieldHelper to write solr compatible numeric fields without the solr dependency.
Added:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieDateField.java
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieField.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1086651&r1=1086650&r2=1086651&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Mar 29 17:41:58 2011
@@ -333,6 +333,10 @@ New features
* LUCENE-2862: Added TermsEnum.totalTermFreq() and
Terms.getSumTotalTermFreq(). (Mike McCandless, Robert Muir)
+
+* LUCENE-3001: Added TrieFieldHelper to write solr compatible numeric
+ fields without the solr dependency. (ryan)
+
Optimizations
Added: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java?rev=1086651&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java (added)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/TrieFieldHelper.java Tue Mar 29 17:41:58 2011
@@ -0,0 +1,163 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.util;
+
+import java.util.Date;
+
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
+
+/**
+ * Helper class to make TrieFields compatible with ones written in solr
+ */
+public class TrieFieldHelper {
+
+ private TrieFieldHelper() {}
+
+ public static class FieldInfo {
+ public int precisionStep = 8; // same as solr default
+ public boolean store = true;
+ public boolean index = true;
+ public boolean omitNorms = true;
+ public boolean omitTF = true;
+ }
+
+ //----------------------------------------------
+ // Create Field
+ //----------------------------------------------
+
+ private static Fieldable createField(String name, byte[] arr, TokenStream ts, FieldInfo info, float boost) {
+
+ Field f;
+ if (info.store) {
+ f = new Field(name, arr);
+ if (info.index) f.setTokenStream(ts);
+ } else {
+ f = new Field(name, ts);
+ }
+
+ // term vectors aren't supported
+ f.setOmitNorms(info.omitNorms);
+ f.setOmitTermFreqAndPositions(info.omitTF);
+ f.setBoost(boost);
+ return f;
+ }
+
+ public static Fieldable createIntField(String name, int value, FieldInfo info, float boost) {
+
+ byte[] arr=null;
+ TokenStream ts=null;
+
+ if (info.store) arr = TrieFieldHelper.toArr(value);
+ if (info.index) ts = new NumericTokenStream(info.precisionStep).setIntValue(value);
+
+ return createField(name, arr, ts, info, boost);
+ }
+
+ public static Fieldable createFloatField(String name, float value, FieldInfo info, float boost) {
+
+ byte[] arr=null;
+ TokenStream ts=null;
+
+ if (info.store) arr = TrieFieldHelper.toArr(value);
+ if (info.index) ts = new NumericTokenStream(info.precisionStep).setFloatValue(value);
+
+ return createField(name, arr, ts, info, boost);
+ }
+
+ public static Fieldable createLongField(String name, long value, FieldInfo info, float boost) {
+
+ byte[] arr=null;
+ TokenStream ts=null;
+
+ if (info.store) arr = TrieFieldHelper.toArr(value);
+ if (info.index) ts = new NumericTokenStream(info.precisionStep).setLongValue(value);
+
+ return createField(name, arr, ts, info, boost);
+ }
+
+ public static Fieldable createDoubleField(String name, double value, FieldInfo info, float boost) {
+
+ byte[] arr=null;
+ TokenStream ts=null;
+
+ if (info.store) arr = TrieFieldHelper.toArr(value);
+ if (info.index) ts = new NumericTokenStream(info.precisionStep).setDoubleValue(value);
+
+ return createField(name, arr, ts, info, boost);
+ }
+
+ public static Fieldable createDateField(String name, Date value, FieldInfo info, float boost) {
+ // TODO, make sure the date is within long range!
+ return createLongField(name, value.getTime(), info, boost);
+ }
+
+
+ //----------------------------------------------
+ // number <=> byte[]
+ //----------------------------------------------
+
+ public static int toInt(byte[] arr) {
+ return (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
+ }
+
+ public static long toLong(byte[] arr) {
+ int high = (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
+ int low = (arr[4]<<24) | ((arr[5]&0xff)<<16) | ((arr[6]&0xff)<<8) | (arr[7]&0xff);
+ return (((long)high)<<32) | (low&0x0ffffffffL);
+ }
+
+ public static float toFloat(byte[] arr) {
+ return Float.intBitsToFloat(toInt(arr));
+ }
+
+ public static double toDouble(byte[] arr) {
+ return Double.longBitsToDouble(toLong(arr));
+ }
+
+ public static byte[] toArr(int val) {
+ byte[] arr = new byte[4];
+ arr[0] = (byte)(val>>>24);
+ arr[1] = (byte)(val>>>16);
+ arr[2] = (byte)(val>>>8);
+ arr[3] = (byte)(val);
+ return arr;
+ }
+
+ public static byte[] toArr(long val) {
+ byte[] arr = new byte[8];
+ arr[0] = (byte)(val>>>56);
+ arr[1] = (byte)(val>>>48);
+ arr[2] = (byte)(val>>>40);
+ arr[3] = (byte)(val>>>32);
+ arr[4] = (byte)(val>>>24);
+ arr[5] = (byte)(val>>>16);
+ arr[6] = (byte)(val>>>8);
+ arr[7] = (byte)(val);
+ return arr;
+ }
+
+ public static byte[] toArr(float val) {
+ return toArr(Float.floatToRawIntBits(val));
+ }
+
+ public static byte[] toArr(double val) {
+ return toArr(Double.doubleToRawLongBits(val));
+ }
+}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieDateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieDateField.java?rev=1086651&r1=1086650&r2=1086651&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieDateField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieDateField.java Tue Mar 29 17:41:58 2011
@@ -27,7 +27,6 @@ import org.apache.solr.search.function.*
import org.apache.solr.search.QParser;
import org.apache.solr.response.TextResponseWriter;
import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Field;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Query;
@@ -36,8 +35,7 @@ import org.apache.lucene.search.cache.Ca
import org.apache.lucene.search.cache.LongValuesCreator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.util.TrieFieldHelper;
import java.util.Map;
import java.util.Date;
@@ -68,7 +66,7 @@ public class TrieDateField extends DateF
public Date toObject(Fieldable f) {
byte[] arr = f.getBinaryValue();
if (arr==null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,TrieField.badFieldString(f));
- return new Date(TrieField.toLong(arr));
+ return new Date(TrieFieldHelper.toLong(arr));
}
@Override
@@ -107,7 +105,7 @@ public class TrieDateField extends DateF
return;
}
- writer.writeDate(name,new Date(TrieField.toLong(arr)));
+ writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr)));
}
@Override
@@ -146,7 +144,7 @@ public class TrieDateField extends DateF
public String toExternal(Fieldable f) {
byte[] arr = f.getBinaryValue();
if (arr==null) return TrieField.badFieldString(f);
- return super.toExternal(new Date(TrieField.toLong(arr)));
+ return super.toExternal(new Date(TrieFieldHelper.toLong(arr)));
}
@Override
@@ -167,44 +165,6 @@ public class TrieDateField extends DateF
return readableToIndexed(storedToReadable(f));
}
- @Override
- public Fieldable createField(SchemaField field, Object value, float boost) {
- boolean indexed = field.indexed();
- boolean stored = field.stored();
-
- if (!indexed && !stored) {
- if (log.isTraceEnabled())
- log.trace("Ignoring unindexed/unstored field: " + field);
- return null;
- }
-
- int ps = precisionStep;
-
- byte[] arr=null;
- TokenStream ts=null;
-
- long time = (value instanceof Date)
- ? ((Date)value).getTime()
- : super.parseMath(null, value.toString()).getTime();
-
- if (stored) arr = TrieField.toArr(time);
- if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
-
- Field f;
- if (stored) {
- f = new Field(field.getName(), arr);
- if (indexed) f.setTokenStream(ts);
- } else {
- f = new Field(field.getName(), ts);
- }
-
- // term vectors aren't supported
-
- f.setOmitNorms(field.omitNorms());
- f.setOmitTermFreqAndPositions(field.omitTf());
- f.setBoost(boost);
- return f;
- }
@Override
public Query getRangeQuery(QParser parser, SchemaField field, String min, String max, boolean minInclusive, boolean maxInclusive) {
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieField.java?rev=1086651&r1=1086650&r2=1086651&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/schema/TrieField.java Tue Mar 29 17:41:58 2011
@@ -17,7 +17,6 @@
package org.apache.solr.schema;
import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Field;
import org.apache.lucene.search.*;
import org.apache.lucene.search.cache.CachedArrayCreator;
import org.apache.lucene.search.cache.DoubleValuesCreator;
@@ -26,8 +25,7 @@ import org.apache.lucene.search.cache.In
import org.apache.lucene.search.cache.LongValuesCreator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.util.TrieFieldHelper;
import org.apache.noggit.CharArr;
import org.apache.solr.analysis.*;
import org.apache.solr.common.SolrException;
@@ -68,6 +66,7 @@ public class TrieField extends FieldType
protected TrieTypes type;
protected Object missingValue;
+
/**
* Used for handling date types following the same semantics as DateField
*/
@@ -107,15 +106,15 @@ public class TrieField extends FieldType
if (arr==null) return badFieldString(f);
switch (type) {
case INTEGER:
- return toInt(arr);
+ return TrieFieldHelper.toInt(arr);
case FLOAT:
- return toFloat(arr);
+ return TrieFieldHelper.toFloat(arr);
case LONG:
- return toLong(arr);
+ return TrieFieldHelper.toLong(arr);
case DOUBLE:
- return toDouble(arr);
+ return TrieFieldHelper.toDouble(arr);
case DATE:
- return new Date(toLong(arr));
+ return new Date(TrieFieldHelper.toLong(arr));
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
@@ -207,19 +206,19 @@ public class TrieField extends FieldType
}
switch (type) {
case INTEGER:
- writer.writeInt(name,toInt(arr));
+ writer.writeInt(name,TrieFieldHelper.toInt(arr));
break;
case FLOAT:
- writer.writeFloat(name,toFloat(arr));
+ writer.writeFloat(name,TrieFieldHelper.toFloat(arr));
break;
case LONG:
- writer.writeLong(name,toLong(arr));
+ writer.writeLong(name,TrieFieldHelper.toLong(arr));
break;
case DOUBLE:
- writer.writeDouble(name,toDouble(arr));
+ writer.writeDouble(name,TrieFieldHelper.toDouble(arr));
break;
case DATE:
- writer.writeDate(name,new Date(toLong(arr)));
+ writer.writeDate(name,new Date(TrieFieldHelper.toLong(arr)));
break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
@@ -293,55 +292,6 @@ public class TrieField extends FieldType
}
- static int toInt(byte[] arr) {
- return (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
- }
-
- static long toLong(byte[] arr) {
- int high = (arr[0]<<24) | ((arr[1]&0xff)<<16) | ((arr[2]&0xff)<<8) | (arr[3]&0xff);
- int low = (arr[4]<<24) | ((arr[5]&0xff)<<16) | ((arr[6]&0xff)<<8) | (arr[7]&0xff);
- return (((long)high)<<32) | (low&0x0ffffffffL);
- }
-
- static float toFloat(byte[] arr) {
- return Float.intBitsToFloat(toInt(arr));
- }
-
- static double toDouble(byte[] arr) {
- return Double.longBitsToDouble(toLong(arr));
- }
-
- static byte[] toArr(int val) {
- byte[] arr = new byte[4];
- arr[0] = (byte)(val>>>24);
- arr[1] = (byte)(val>>>16);
- arr[2] = (byte)(val>>>8);
- arr[3] = (byte)(val);
- return arr;
- }
-
- static byte[] toArr(long val) {
- byte[] arr = new byte[8];
- arr[0] = (byte)(val>>>56);
- arr[1] = (byte)(val>>>48);
- arr[2] = (byte)(val>>>40);
- arr[3] = (byte)(val>>>32);
- arr[4] = (byte)(val>>>24);
- arr[5] = (byte)(val>>>16);
- arr[6] = (byte)(val>>>8);
- arr[7] = (byte)(val);
- return arr;
- }
-
- static byte[] toArr(float val) {
- return toArr(Float.floatToRawIntBits(val));
- }
-
- static byte[] toArr(double val) {
- return toArr(Double.doubleToRawLongBits(val));
- }
-
-
@Override
public String storedToReadable(Fieldable f) {
return toExternal(f);
@@ -396,15 +346,15 @@ public class TrieField extends FieldType
if (arr==null) return badFieldString(f);
switch (type) {
case INTEGER:
- return Integer.toString(toInt(arr));
+ return Integer.toString(TrieFieldHelper.toInt(arr));
case FLOAT:
- return Float.toString(toFloat(arr));
+ return Float.toString(TrieFieldHelper.toFloat(arr));
case LONG:
- return Long.toString(toLong(arr));
+ return Long.toString(TrieFieldHelper.toLong(arr));
case DOUBLE:
- return Double.toString(toDouble(arr));
+ return Double.toString(TrieFieldHelper.toDouble(arr));
case DATE:
- return dateField.formatDate(new Date(toLong(arr)));
+ return dateField.formatDate(new Date(TrieFieldHelper.toLong(arr)));
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
@@ -483,75 +433,53 @@ public class TrieField extends FieldType
@Override
public Fieldable createField(SchemaField field, Object value, float boost) {
- boolean indexed = field.indexed();
- boolean stored = field.stored();
-
- if (!indexed && !stored) {
+ TrieFieldHelper.FieldInfo info = new TrieFieldHelper.FieldInfo();
+ info.index = field.indexed();
+ info.store = field.stored();
+ info.precisionStep = precisionStep;
+ info.omitNorms = field.omitNorms();
+ info.omitTF = field.omitTf();
+
+ if (!info.index && !info.store) {
if (log.isTraceEnabled())
log.trace("Ignoring unindexed/unstored field: " + field);
return null;
}
- int ps = precisionStep;
-
- byte[] arr=null;
- TokenStream ts=null;
- // String indexedVal = indexed && precisionStep==0 ? readableToIndexed(externalVal) : null;
-
switch (type) {
case INTEGER:
int i = (value instanceof Number)
? ((Number)value).intValue()
: Integer.parseInt(value.toString());
- if (stored) arr = toArr(i);
- if (indexed) ts = new NumericTokenStream(ps).setIntValue(i);
- break;
+ return TrieFieldHelper.createIntField(field.getName(), i, info, boost);
+
case FLOAT:
float f = (value instanceof Number)
? ((Number)value).floatValue()
: Float.parseFloat(value.toString());
- if (stored) arr = toArr(f);
- if (indexed) ts = new NumericTokenStream(ps).setFloatValue(f);
- break;
+ return TrieFieldHelper.createFloatField(field.getName(), f, info, boost);
+
case LONG:
long l = (value instanceof Number)
? ((Number)value).longValue()
: Long.parseLong(value.toString());
- if (stored) arr = toArr(l);
- if (indexed) ts = new NumericTokenStream(ps).setLongValue(l);
- break;
+ return TrieFieldHelper.createLongField(field.getName(), l, info, boost);
+
case DOUBLE:
double d = (value instanceof Number)
? ((Number)value).doubleValue()
: Double.parseDouble(value.toString());
- if (stored) arr = toArr(d);
- if (indexed) ts = new NumericTokenStream(ps).setDoubleValue(d);
- break;
+ return TrieFieldHelper.createDoubleField(field.getName(), d, info, boost);
+
case DATE:
- long time = (value instanceof Date)
- ? ((Date)value).getTime()
- : dateField.parseMath(null, value.toString()).getTime();
- if (stored) arr = toArr(time);
- if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
- break;
+ Date date = (value instanceof Date)
+ ? ((Date)value)
+ : dateField.parseMath(null, value.toString());
+ return TrieFieldHelper.createDateField(field.getName(), date, info, boost);
+
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
}
-
- Field f;
- if (stored) {
- f = new Field(field.getName(), arr);
- if (indexed) f.setTokenStream(ts);
- } else {
- f = new Field(field.getName(), ts);
- }
-
- // term vectors aren't supported
-
- f.setOmitNorms(field.omitNorms());
- f.setOmitTermFreqAndPositions(field.omitTf());
- f.setBoost(boost);
- return f;
}
public enum TrieTypes {