You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/08/27 15:27:12 UTC
svn commit: r1162347 [4/15] - in /lucene/dev/trunk: lucene/ lucene/contrib/
lucene/contrib/demo/src/java/org/apache/lucene/demo/
lucene/contrib/demo/src/java/org/apache/lucene/demo/xmlparser/
lucene/contrib/highlighter/src/java/org/apache/lucene/search...
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java Sat Aug 27 13:27:01 2011
@@ -20,16 +20,13 @@ import java.io.Reader;
import java.util.Comparator;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.values.PerDocFieldValues;
import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.util.BytesRef;
/**
* <p>
- * This class provides a {@link AbstractField} that enables storing of typed
+ * This class provides a {@link Field} that enables storing of typed
* per-document values for scoring, sorting or value retrieval. Here's an
* example usage, adding an int value:
*
@@ -54,16 +51,14 @@ import org.apache.lucene.util.BytesRef;
* </pre>
*
* <p>
- * If doc values are stored in addition to an indexed ({@link Index}) or stored
- * ({@link Store}) value it's recommended to use the {@link IndexDocValuesField}'s
- * {@link #set(AbstractField)} API:
+ * If doc values are stored in addition to an indexed ({@link FieldType#setIndexed(boolean)}) or stored
+ * ({@link FieldType#setStored(boolean)}) value it's recommended to pass the appropriate {@link FieldType}
+ * when creating the field:
*
* <pre>
- * IndexDocValuesField field = new IndexDocValuesField(name);
- * Field indexedField = new Field(name, stringValue, Stored.NO, Indexed.ANALYZED);
+ * IndexDocValuesField field = new IndexDocValuesField(name, StringField.TYPE_STORED);
* Document document = new Document();
- * document.add(indexedField);
- * field.set(indexedField);
+ * document.add(field);
* for(all documents) {
* ...
* field.setInt(value)
@@ -73,7 +68,8 @@ import org.apache.lucene.util.BytesRef;
* </pre>
*
* */
-public class IndexDocValuesField extends AbstractField implements PerDocFieldValues {
+// TODO: maybe rename to DocValuesField?
+public class IndexDocValuesField extends Field implements PerDocFieldValues {
protected BytesRef bytes;
protected double doubleValue;
@@ -85,21 +81,27 @@ public class IndexDocValuesField extends
* Creates a new {@link IndexDocValuesField} with the given name.
*/
public IndexDocValuesField(String name) {
- super(name, Store.NO, Index.NO, TermVector.NO);
- setDocValues(this);
+ this(name, new FieldType());
}
- /**
- * Creates a {@link IndexDocValuesField} prototype
- */
- IndexDocValuesField() {
- this("");
+ public IndexDocValuesField(String name, FieldType type) {
+ this(name, type, null);
+ }
+
+ public IndexDocValuesField(String name, FieldType type, String value) {
+ super(name, type);
+ fieldsData = value;
+ }
+
+ @Override
+ public PerDocFieldValues docValues() {
+ return this;
}
/**
* Sets the given <code>long</code> value and sets the field's {@link ValueType} to
* {@link ValueType#VAR_INTS} unless already set. If you want to change the
- * default type use {@link #setType(ValueType)}.
+ * default type use {@link #setDocValuesType(ValueType)}.
*/
public void setInt(long value) {
setInt(value, false);
@@ -124,7 +126,7 @@ public class IndexDocValuesField extends
/**
* Sets the given <code>int</code> value and sets the field's {@link ValueType} to
* {@link ValueType#VAR_INTS} unless already set. If you want to change the
- * default type use {@link #setType(ValueType)}.
+ * default type use {@link #setDocValuesType(ValueType)}.
*/
public void setInt(int value) {
setInt(value, false);
@@ -149,7 +151,7 @@ public class IndexDocValuesField extends
/**
* Sets the given <code>short</code> value and sets the field's {@link ValueType} to
* {@link ValueType#VAR_INTS} unless already set. If you want to change the
- * default type use {@link #setType(ValueType)}.
+ * default type use {@link #setDocValuesType(ValueType)}.
*/
public void setInt(short value) {
setInt(value, false);
@@ -174,11 +176,12 @@ public class IndexDocValuesField extends
/**
* Sets the given <code>byte</code> value and sets the field's {@link ValueType} to
* {@link ValueType#VAR_INTS} unless already set. If you want to change the
- * default type use {@link #setType(ValueType)}.
+ * default type use {@link #setDocValuesType(ValueType)}.
*/
public void setInt(byte value) {
setInt(value, false);
}
+
/**
* Sets the given <code>byte</code> value as a 8 bit signed integer.
*
@@ -198,7 +201,7 @@ public class IndexDocValuesField extends
/**
* Sets the given <code>float</code> value and sets the field's {@link ValueType}
* to {@link ValueType#FLOAT_32} unless already set. If you want to
- * change the type use {@link #setType(ValueType)}.
+ * change the type use {@link #setDocValuesType(ValueType)}.
*/
public void setFloat(float value) {
if (type == null) {
@@ -210,7 +213,7 @@ public class IndexDocValuesField extends
/**
* Sets the given <code>double</code> value and sets the field's {@link ValueType}
* to {@link ValueType#FLOAT_64} unless already set. If you want to
- * change the default type use {@link #setType(ValueType)}.
+ * change the default type use {@link #setDocValuesType(ValueType)}.
*/
public void setFloat(double value) {
if (type == null) {
@@ -241,7 +244,7 @@ public class IndexDocValuesField extends
if (value == null) {
throw new IllegalArgumentException("value must not be null");
}
- setType(type);
+ setDocValuesType(type);
if (bytes == null) {
bytes = new BytesRef(value);
} else {
@@ -289,7 +292,7 @@ public class IndexDocValuesField extends
/**
* Sets the {@link ValueType} for this field.
*/
- public void setType(ValueType type) {
+ public void setDocValuesType(ValueType type) {
if (type == null) {
throw new IllegalArgumentException("Type must not be null");
}
@@ -297,13 +300,6 @@ public class IndexDocValuesField extends
}
/**
- * Returns the field's {@link ValueType}
- */
- public ValueType type() {
- return type;
- }
-
- /**
* Returns always <code>null</code>
*/
public Reader readerValue() {
@@ -313,36 +309,54 @@ public class IndexDocValuesField extends
/**
* Returns always <code>null</code>
*/
- public String stringValue() {
+ public TokenStream tokenStreamValue() {
return null;
}
- /**
- * Returns always <code>null</code>
- */
- public TokenStream tokenStreamValue() {
- return null;
+ @Override
+ public ValueType docValuesType() {
+ return type;
}
- /**
- * Sets this {@link IndexDocValuesField} to the given {@link AbstractField} and
- * returns the given field. Any modifications to this instance will be visible
- * to the given field.
- */
- public <T extends AbstractField> T set(T field) {
- field.setDocValues(this);
- return field;
+ @Override
+ public String toString() {
+ final String value;
+ switch (type) {
+ case BYTES_FIXED_DEREF:
+ case BYTES_FIXED_SORTED:
+ case BYTES_FIXED_STRAIGHT:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ value = "bytes:bytes.utf8ToString();";
+ break;
+ case VAR_INTS:
+ value = "int:" + longValue;
+ break;
+ case FLOAT_32:
+ value = "float32:" + doubleValue;
+ break;
+ case FLOAT_64:
+ value = "float64:" + doubleValue;
+ break;
+ default:
+ throw new IllegalArgumentException("unknown type: " + type);
+ }
+ return "<" + name() + ": IndexDocValuesField " + value + ">";
}
/**
- * Sets a new {@link PerDocFieldValues} instance on the given field with the
- * given type and returns it.
- *
+ * Returns an IndexDocValuesField holding the value from
+ * the provided string field, as the specified type. The
+ * incoming field must have a string value. The name, {@link
+ * FieldType} and string value are carried over from the
+ * incoming Field.
*/
- public static <T extends AbstractField> T set(T field, ValueType type) {
- if (field instanceof IndexDocValuesField)
- return field;
- final IndexDocValuesField valField = new IndexDocValuesField();
+ public static IndexDocValuesField build(Field field, ValueType type) {
+ if (field instanceof IndexDocValuesField) {
+ return (IndexDocValuesField) field;
+ }
+ final IndexDocValuesField valField = new IndexDocValuesField(field.name(), field.getFieldType(), field.stringValue());
switch (type) {
case BYTES_FIXED_DEREF:
case BYTES_FIXED_SORTED:
@@ -350,9 +364,7 @@ public class IndexDocValuesField extends
case BYTES_VAR_DEREF:
case BYTES_VAR_SORTED:
case BYTES_VAR_STRAIGHT:
- BytesRef ref = field.isBinary() ? new BytesRef(field.getBinaryValue(),
- field.getBinaryOffset(), field.getBinaryLength()) : new BytesRef(
- field.stringValue());
+ BytesRef ref = field.isBinary() ? field.binaryValue() : new BytesRef(field.stringValue());
valField.setBytes(ref, type);
break;
case VAR_INTS:
@@ -367,7 +379,6 @@ public class IndexDocValuesField extends
default:
throw new IllegalArgumentException("unknown type: " + type);
}
- return valField.set(field);
+ return valField;
}
-
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/NumericField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/NumericField.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/NumericField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/NumericField.java Sat Aug 27 13:27:01 2011
@@ -22,28 +22,30 @@ import java.io.Reader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.NumericTokenStream;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.document.NumericField.DataType;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.search.NumericRangeQuery; // javadocs
import org.apache.lucene.search.NumericRangeFilter; // javadocs
import org.apache.lucene.search.FieldCache; // javadocs
/**
- * <p>This class provides a {@link Field} that enables indexing
- * of numeric values for efficient range filtering and
- * sorting. Here's an example usage, adding an int value:
+ * <p>
+ * This class provides a {@link Field} that enables indexing of numeric values
+ * for efficient range filtering and sorting. Here's an example usage, adding an
+ * int value:
+ *
* <pre>
- * document.add(new NumericField(name).setIntValue(value));
+ * document.add(new NumericField(name).setIntValue(value));
* </pre>
- *
- * For optimal performance, re-use the
- * <code>NumericField</code> and {@link Document} instance for more than
- * one document:
- *
+ *
+ * For optimal performance, re-use the <code>NumericField</code> and
+ * {@link Document} instance for more than one document:
+ *
* <pre>
* NumericField field = new NumericField(name);
* Document document = new Document();
* document.add(field);
- *
+ *
* for(all documents) {
* ...
* field.setIntValue(value)
@@ -74,7 +76,7 @@ import org.apache.lucene.search.FieldCac
*
* <p>By default, a <code>NumericField</code>'s value is not stored but
* is indexed for range filtering and sorting. You can use
- * the {@link #NumericField(String,Field.Store,boolean)}
+ * the {@link #NumericField(String, FieldType)}
* constructor if you need to change these defaults.</p>
*
* <p>You may add the same field name as a <code>NumericField</code> to
@@ -102,7 +104,7 @@ import org.apache.lucene.search.FieldCac
* default value, 4, was selected for a reasonable tradeoff
* of disk space consumption versus performance. You can
* use the expert constructor {@link
- * #NumericField(String,int,Field.Store,boolean)} if you'd
+ * #NumericField(String,int,FieldType)} if you'd
* like to change the value. Note that you must also
* specify a congruent value when creating {@link
* NumericRangeQuery} or {@link NumericRangeFilter}.
@@ -129,94 +131,136 @@ import org.apache.lucene.search.FieldCac
*
* @since 2.9
*/
-public final class NumericField extends AbstractField {
-
+public final class NumericField extends Field {
+
/** Data type of the value in {@link NumericField}.
* @since 3.2
*/
public static enum DataType { INT, LONG, FLOAT, DOUBLE }
+ public static final FieldType TYPE_UNSTORED = new FieldType();
+ public static final FieldType TYPE_STORED = new FieldType();
+ static {
+ TYPE_UNSTORED.setIndexed(true);
+ TYPE_UNSTORED.setTokenized(true);
+ TYPE_UNSTORED.setOmitNorms(true);
+ TYPE_UNSTORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_UNSTORED.freeze();
+
+ TYPE_STORED.setIndexed(true);
+ TYPE_STORED.setStored(true);
+ TYPE_STORED.setTokenized(true);
+ TYPE_STORED.setOmitNorms(true);
+ TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_STORED.freeze();
+ }
+
+ //public static enum DataType { INT, LONG, FLOAT, DOUBLE }
+
+ private DataType dataType;
private transient NumericTokenStream numericTS;
- private DataType type;
private final int precisionStep;
-
+
/**
- * Creates a field for numeric values using the default <code>precisionStep</code>
- * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The instance is not yet initialized with
- * a numeric value, before indexing a document containing this field,
- * set a value using the various set<em>???</em>Value() methods.
- * This constructor creates an indexed, but not stored field.
- * @param name the field name
+ * Creates a field for numeric values using the default
+ * <code>precisionStep</code> {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * The instance is not yet initialized with a numeric value, before indexing a
+ * document containing this field, set a value using the various set
+ * <em>???</em>Value() methods. This constructor creates an indexed, but not
+ * stored field.
+ *
+ * @param name
+ * the field name
*/
public NumericField(String name) {
- this(name, NumericUtils.PRECISION_STEP_DEFAULT, Field.Store.NO, true);
+ this(name, NumericUtils.PRECISION_STEP_DEFAULT, NumericField.TYPE_UNSTORED);
}
/**
- * Creates a field for numeric values using the default <code>precisionStep</code>
- * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The instance is not yet initialized with
- * a numeric value, before indexing a document containing this field,
- * set a value using the various set<em>???</em>Value() methods.
- * @param name the field name
- * @param store if the field should be stored, {@link Document#getFieldable}
- * then returns {@code NumericField} instances on search results.
- * @param index if the field should be indexed using {@link NumericTokenStream}
+ * Creates a field for numeric values using the default
+ * <code>precisionStep</code> {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * The instance is not yet initialized with a numeric value, before indexing a
+ * document containing this field, set a value using the various set
+ * <em>???</em>Value() methods.
+ *
+ * @param name
+ * the field name
+ * @param type
+ * if the defualt field should be altered, e.g. stored,
+ * {@link Document#getField} then returns {@code NumericField}
+ * instances on search results, or indexed using
+ * {@link NumericTokenStream}
*/
- public NumericField(String name, Field.Store store, boolean index) {
- this(name, NumericUtils.PRECISION_STEP_DEFAULT, store, index);
+ public NumericField(String name, FieldType type) {
+ this(name, NumericUtils.PRECISION_STEP_DEFAULT, type);
}
/**
* Creates a field for numeric values with the specified
- * <code>precisionStep</code>. The instance is not yet initialized with
- * a numeric value, before indexing a document containing this field,
- * set a value using the various set<em>???</em>Value() methods.
- * This constructor creates an indexed, but not stored field.
- * @param name the field name
- * @param precisionStep the used <a href="../search/NumericRangeQuery.html#precisionStepDesc">precision step</a>
+ * <code>precisionStep</code>. The instance is not yet initialized with a
+ * numeric value, before indexing a document containing this field, set a
+ * value using the various set<em>???</em>Value() methods. This constructor
+ * creates an indexed, but not stored field.
+ *
+ * @param name
+ * the field name
+ * @param precisionStep
+ * the used <a
+ * href="../search/NumericRangeQuery.html#precisionStepDesc"
+ * >precision step</a>
*/
public NumericField(String name, int precisionStep) {
- this(name, precisionStep, Field.Store.NO, true);
+ this(name, precisionStep, NumericField.TYPE_UNSTORED);
}
-
+
/**
* Creates a field for numeric values with the specified
- * <code>precisionStep</code>. The instance is not yet initialized with
- * a numeric value, before indexing a document containing this field,
- * set a value using the various set<em>???</em>Value() methods.
- * @param name the field name
- * @param precisionStep the used <a href="../search/NumericRangeQuery.html#precisionStepDesc">precision step</a>
- * @param store if the field should be stored, {@link Document#getFieldable}
- * then returns {@code NumericField} instances on search results.
- * @param index if the field should be indexed using {@link NumericTokenStream}
+ * <code>precisionStep</code>. The instance is not yet initialized with a
+ * numeric value, before indexing a document containing this field, set a
+ * value using the various set<em>???</em>Value() methods.
+ *
+ * @param name
+ * the field name
+ * @param precisionStep
+ * the used <a
+ * href="../search/NumericRangeQuery.html#precisionStepDesc"
+ * >precision step</a>
+ * @param type
+ * if the defualt field should be altered, e.g. stored,
+ * {@link Document#getField} then returns {@code NumericField}
+ * instances on search results, or indexed using
+ * {@link NumericTokenStream}
*/
- public NumericField(String name, int precisionStep, Field.Store store, boolean index) {
- super(name, store, index ? Field.Index.ANALYZED_NO_NORMS : Field.Index.NO, Field.TermVector.NO);
+ public NumericField(String name, int precisionStep, FieldType type) {
+ super(name, type);
this.precisionStep = precisionStep;
- setIndexOptions(IndexOptions.DOCS_ONLY);
}
-
+
/** Returns a {@link NumericTokenStream} for indexing the numeric value. */
- public TokenStream tokenStreamValue() {
- if (!isIndexed())
- return null;
+ public TokenStream tokenStreamValue() {
+ if (!indexed()) return null;
if (numericTS == null) {
- // lazy init the TokenStream as it is heavy to instantiate (attributes,...),
+ // lazy init the TokenStream as it is heavy to instantiate
+ // (attributes,...),
// if not needed (stored field loading)
numericTS = new NumericTokenStream(precisionStep);
// initialize value in TokenStream
if (fieldsData != null) {
- assert type != null;
+ assert dataType != null;
final Number val = (Number) fieldsData;
- switch (type) {
+ switch (dataType) {
case INT:
- numericTS.setIntValue(val.intValue()); break;
+ numericTS.setIntValue(val.intValue());
+ break;
case LONG:
- numericTS.setLongValue(val.longValue()); break;
+ numericTS.setLongValue(val.longValue());
+ break;
case FLOAT:
- numericTS.setFloatValue(val.floatValue()); break;
+ numericTS.setFloatValue(val.floatValue());
+ break;
case DOUBLE:
- numericTS.setDoubleValue(val.doubleValue()); break;
+ numericTS.setDoubleValue(val.doubleValue());
+ break;
default:
assert false : "Should never get here";
}
@@ -226,26 +270,27 @@ public final class NumericField extends
}
/** Returns always <code>null</code> for numeric fields */
- @Override
- public byte[] getBinaryValue(byte[] result){
- return null;
- }
-
- /** Returns always <code>null</code> for numeric fields */
public Reader readerValue() {
return null;
}
-
- /** Returns the numeric value as a string. This format is also returned if you call {@link Document#get(String)}
- * on search results. It is recommended to use {@link Document#getFieldable} instead
- * that returns {@code NumericField} instances. You can then use {@link #getNumericValue}
- * to return the stored value. */
- public String stringValue() {
+
+ /**
+ * Returns the numeric value as a string. It is recommended to
+ * use {@link Document#getField} instead that returns {@code NumericField}
+ * instances. You can then use {@link #numericValue} to return the stored
+ * value.
+ */
+ @Override
+ public String stringValue() {
return (fieldsData == null) ? null : fieldsData.toString();
}
- /** Returns the current numeric value as a subclass of {@link Number}, <code>null</code> if not yet initialized. */
- public Number getNumericValue() {
+ /**
+ * Returns the current numeric value as a subclass of {@link Number},
+ * <code>null</code> if not yet initialized.
+ */
+ @Override
+ public Number numericValue() {
return (Number) fieldsData;
}
@@ -254,63 +299,79 @@ public final class NumericField extends
return precisionStep;
}
- /** Returns the data type of the current value, {@code null} if not yet set.
+ /**
+ * Returns the data type of the current value, {@code null} if not yet set.
+ *
* @since 3.2
*/
- public DataType getDataType() {
- return type;
+ @Override
+ public DataType numericDataType() {
+ return dataType;
}
-
+
+ @Override
+ public boolean numeric() {
+ return true;
+ }
+
/**
* Initializes the field with the supplied <code>long</code> value.
- * @param value the numeric value
+ *
+ * @param value
+ * the numeric value
* @return this instance, because of this you can use it the following way:
- * <code>document.add(new NumericField(name, precisionStep).setLongValue(value))</code>
+ * <code>document.add(new NumericField(name, precisionStep).setLongValue(value))</code>
*/
public NumericField setLongValue(final long value) {
if (numericTS != null) numericTS.setLongValue(value);
fieldsData = Long.valueOf(value);
- type = DataType.LONG;
+ dataType = DataType.LONG;
return this;
}
/**
* Initializes the field with the supplied <code>int</code> value.
- * @param value the numeric value
+ *
+ * @param value
+ * the numeric value
* @return this instance, because of this you can use it the following way:
- * <code>document.add(new NumericField(name, precisionStep).setIntValue(value))</code>
+ * <code>document.add(new NumericField(name, precisionStep).setIntValue(value))</code>
*/
public NumericField setIntValue(final int value) {
if (numericTS != null) numericTS.setIntValue(value);
fieldsData = Integer.valueOf(value);
- type = DataType.INT;
+ dataType = DataType.INT;
return this;
}
/**
* Initializes the field with the supplied <code>double</code> value.
- * @param value the numeric value
+ *
+ * @param value
+ * the numeric value
* @return this instance, because of this you can use it the following way:
- * <code>document.add(new NumericField(name, precisionStep).setDoubleValue(value))</code>
+ * <code>document.add(new NumericField(name, precisionStep).setDoubleValue(value))</code>
*/
public NumericField setDoubleValue(final double value) {
if (numericTS != null) numericTS.setDoubleValue(value);
fieldsData = Double.valueOf(value);
- type = DataType.DOUBLE;
+ dataType = DataType.DOUBLE;
return this;
}
/**
* Initializes the field with the supplied <code>float</code> value.
- * @param value the numeric value
+ *
+ * @param value
+ * the numeric value
* @return this instance, because of this you can use it the following way:
- * <code>document.add(new NumericField(name, precisionStep).setFloatValue(value))</code>
+ * <code>document.add(new NumericField(name, precisionStep).setFloatValue(value))</code>
*/
public NumericField setFloatValue(final float value) {
if (numericTS != null) numericTS.setFloatValue(value);
fieldsData = Float.valueOf(value);
- type = DataType.FLOAT;
+ dataType = DataType.FLOAT;
return this;
}
-
+
}
Added: lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/StringField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/StringField.java?rev=1162347&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/StringField.java (added)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/StringField.java Sat Aug 27 13:27:01 2011
@@ -0,0 +1,51 @@
+package org.apache.lucene.document;
+
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public final class StringField extends Field {
+
+ public static final FieldType TYPE_UNSTORED = new FieldType();
+ public static final FieldType TYPE_STORED = new FieldType();
+ static {
+ TYPE_UNSTORED.setIndexed(true);
+ TYPE_UNSTORED.setOmitNorms(true);
+ TYPE_UNSTORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_UNSTORED.freeze();
+
+ TYPE_STORED.setIndexed(true);
+ TYPE_STORED.setStored(true);
+ TYPE_STORED.setOmitNorms(true);
+ TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_STORED.freeze();
+ }
+
+ public StringField(String name, String value) {
+ super(name, TYPE_UNSTORED, value);
+ }
+
+ @Override
+ public String stringValue() {
+ return (fieldsData == null) ? null : fieldsData.toString();
+ }
+
+ public boolean isNumeric() {
+ return false;
+ }
+}
Added: lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/TextField.java?rev=1162347&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/TextField.java (added)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/TextField.java Sat Aug 27 13:27:01 2011
@@ -0,0 +1,54 @@
+package org.apache.lucene.document;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+
+public final class TextField extends Field {
+
+ public static final FieldType TYPE_UNSTORED = new FieldType();
+ public static final FieldType TYPE_STORED = new FieldType();
+ static {
+ TYPE_UNSTORED.setIndexed(true);
+ TYPE_UNSTORED.setTokenized(true);
+ TYPE_UNSTORED.freeze();
+
+ TYPE_STORED.setIndexed(true);
+ TYPE_STORED.setStored(true);
+ TYPE_STORED.setTokenized(true);
+ TYPE_STORED.freeze();
+ }
+
+ public TextField(String name, Reader reader) {
+ super(name, TextField.TYPE_UNSTORED, reader);
+ }
+
+ public TextField(String name, String value) {
+ super(name, TextField.TYPE_UNSTORED, value);
+ }
+
+ public TextField(String name, TokenStream stream) {
+ super(name, TextField.TYPE_UNSTORED, stream);
+ }
+
+ public boolean isNumeric() {
+ return false;
+ }
+}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/package.html?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/package.html (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/package.html Sat Aug 27 13:27:01 2011
@@ -22,16 +22,16 @@
<body>
<p>The logical representation of a {@link org.apache.lucene.document.Document} for indexing and searching.</p>
<p>The document package provides the user level logical representation of content to be indexed and searched. The
-package also provides utilities for working with {@link org.apache.lucene.document.Document}s and {@link org.apache.lucene.document.Fieldable}s.</p>
-<h2>Document and Fieldable</h2>
-<p>A {@link org.apache.lucene.document.Document} is a collection of {@link org.apache.lucene.document.Fieldable}s. A
- {@link org.apache.lucene.document.Fieldable} is a logical representation of a user's content that needs to be indexed or stored.
- {@link org.apache.lucene.document.Fieldable}s have a number of properties that tell Lucene how to treat the content (like indexed, tokenized,
- stored, etc.) See the {@link org.apache.lucene.document.Field} implementation of {@link org.apache.lucene.document.Fieldable}
+package also provides utilities for working with {@link org.apache.lucene.document.Document}s and {@link org.apache.lucene.index.IndexableField}s.</p>
+<h2>Document and IndexableField</h2>
+<p>A {@link org.apache.lucene.document.Document} is a collection of {@link org.apache.lucene.index.IndexableField}s. A
+ {@link org.apache.lucene.index.IndexableField} is a logical representation of a user's content that needs to be indexed or stored.
+ {@link org.apache.lucene.index.IndexableField}s have a number of properties that tell Lucene how to treat the content (like indexed, tokenized,
+ stored, etc.) See the {@link org.apache.lucene.document.Field} implementation of {@link org.apache.lucene.index.IndexableField}
for specifics on these properties.
</p>
<p>Note: it is common to refer to {@link org.apache.lucene.document.Document}s having {@link org.apache.lucene.document.Field}s, even though technically they have
-{@link org.apache.lucene.document.Fieldable}s.</p>
+{@link org.apache.lucene.index.IndexableField}s.</p>
<h2>Working with Documents</h2>
<p>First and foremost, a {@link org.apache.lucene.document.Document} is something created by the user application. It is your job
to create Documents based on the content of the files you are working with in your application (Word, txt, PDF, Excel or any other format.)
@@ -45,7 +45,7 @@ package also provides utilities for work
to simplify indexing of numeric values (and also dates) for fast range range queries with {@link org.apache.lucene.search.NumericRangeQuery}
(using a special sortable string representation of numeric values).</p>
<p>The {@link org.apache.lucene.document.FieldSelector} class provides a mechanism to tell Lucene how to load Documents from
-storage. If no FieldSelector is used, all Fieldables on a Document will be loaded. As an example of the FieldSelector usage, consider
+storage. If no FieldSelector is used, all IndexableFields on a Document will be loaded. As an example of the FieldSelector usage, consider
the common use case of
displaying search results on a web page and then having users click through to see the full document. In this scenario, it is often
the case that there are many small fields and one or two large fields (containing the contents of the original file). Before the FieldSelector,
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java Sat Aug 27 13:27:01 2011
@@ -17,6 +17,16 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import org.apache.lucene.document.FieldType; // for javadocs
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
@@ -28,21 +38,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.document.AbstractField; // for javadocs
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.codecs.BlockTreeTermsReader;
-import org.apache.lucene.index.codecs.CodecProvider;
-import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.index.values.ValuesEnum;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.StringHelper;
@@ -189,7 +189,7 @@ public class CheckIndex {
/** True if at least one of the fields in this segment
* has position data
- * @see AbstractField#setIndexOptions(org.apache.lucene.index.FieldInfo.IndexOptions) */
+ * @see FieldType#setIndexOptions(org.apache.lucene.index.FieldInfo.IndexOptions) */
public boolean hasProx;
/** Map that includes certain
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Sat Aug 27 13:27:01 2011
@@ -29,8 +29,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.Lock;
@@ -559,12 +557,11 @@ class DirectoryReader extends IndexReade
return maxDoc;
}
- // inherit javadoc
@Override
- public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+ public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
ensureOpen();
- int i = readerIndex(n); // find segment num
- return subReaders[i].document(n - starts[i], fieldSelector); // dispatch to segment reader
+ int i = readerIndex(docID); // find segment num
+ subReaders[i].document(docID - starts[i], visitor); // dispatch to segment reader
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java Sat Aug 27 13:27:01 2011
@@ -18,11 +18,10 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
abstract class DocFieldConsumerPerField {
/** Processes all occurrences of a single field */
- abstract void processFields(Fieldable[] fields, int count) throws IOException;
+ abstract void processFields(IndexableField[] fields, int count) throws IOException;
abstract void abort();
abstract FieldInfo getFieldInfo();
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java Sat Aug 27 13:27:01 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
final class DocFieldConsumersPerField extends DocFieldConsumerPerField {
@@ -35,7 +34,7 @@ final class DocFieldConsumersPerField ex
}
@Override
- public void processFields(Fieldable[] fields, int count) throws IOException {
+ public void processFields(IndexableField[] fields, int count) throws IOException {
one.processFields(fields, count);
two.processFields(fields, count);
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Sat Aug 27 13:27:01 2011
@@ -22,15 +22,13 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.DocumentsWriterPerThread.DocState;
import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.DocValuesConsumer;
+import org.apache.lucene.index.codecs.PerDocConsumer;
+import org.apache.lucene.index.values.PerDocFieldValues;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.IOUtils;
@@ -199,22 +197,16 @@ final class DocFieldProcessor extends Do
consumer.startDocument();
fieldsWriter.startDocument();
- final Document doc = docState.doc;
-
fieldCount = 0;
final int thisFieldGen = fieldGen++;
- final List<Fieldable> docFields = doc.getFields();
- final int numDocFields = docFields.size();
-
// Absorb any new fields first seen in this document.
// Also absorb any changes to fields we had already
// seen before (eg suddenly turning on norms or
// vectors, etc.):
- for(int i=0;i<numDocFields;i++) {
- Fieldable field = docFields.get(i);
+ for(IndexableField field : docState.doc) {
final String fieldName = field.name();
// Make sure we have a PerField allocated
@@ -231,21 +223,22 @@ final class DocFieldProcessor extends Do
// needs to be more "pluggable" such that if I want
// to have a new "thing" my Fields can do, I can
// easily add it
- FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.isIndexed(), field.isTermVectorStored(),
- field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
- field.getOmitNorms(), false, field.getIndexOptions(), field.docValuesType());
+ FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.indexed(), field.storeTermVectors(),
+ field.storeTermVectorPositions(), field.storeTermVectorOffsets(),
+ field.omitNorms(), false, field.indexOptions(), field.docValuesType());
fp = new DocFieldProcessorPerField(this, fi);
fp.next = fieldHash[hashPos];
fieldHash[hashPos] = fp;
totalFieldCount++;
- if (totalFieldCount >= fieldHash.length/2)
+ if (totalFieldCount >= fieldHash.length/2) {
rehash();
+ }
} else {
- fieldInfos.addOrUpdate(fp.fieldInfo.name, field.isIndexed(), field.isTermVectorStored(),
- field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
- field.getOmitNorms(), false, field.getIndexOptions(), field.docValuesType());
+ fieldInfos.addOrUpdate(fp.fieldInfo.name, field.indexed(), field.storeTermVectors(),
+ field.storeTermVectorPositions(), field.storeTermVectorOffsets(),
+ field.omitNorms(), false, field.indexOptions(), field.docValuesType());
}
if (thisFieldGen != fp.lastGen) {
@@ -266,12 +259,12 @@ final class DocFieldProcessor extends Do
fp.addField(field);
- if (field.isStored()) {
+ if (field.stored()) {
fieldsWriter.addField(field, fp.fieldInfo);
}
- if (field.hasDocValues()) {
- final DocValuesConsumer docValuesConsumer = docValuesConsumer(docState, fp.fieldInfo);
- docValuesConsumer.add(docState.docID, field.getDocValues());
+ final PerDocFieldValues docValues = field.docValues();
+ if (docValues != null) {
+ docValuesConsumer(docState, fp.fieldInfo).add(docState.docID, docValues);
}
}
@@ -339,5 +332,4 @@ final class DocFieldProcessor extends Do
docValues.put(fieldInfo.name, docValuesConsumer);
return docValuesConsumer;
}
-
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java Sat Aug 27 13:27:01 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -34,17 +33,17 @@ final class DocFieldProcessorPerField {
int lastGen = -1;
int fieldCount;
- Fieldable[] fields = new Fieldable[1];
+ IndexableField[] fields = new IndexableField[1];
public DocFieldProcessorPerField(final DocFieldProcessor docFieldProcessor, final FieldInfo fieldInfo) {
this.consumer = docFieldProcessor.consumer.addField(fieldInfo);
this.fieldInfo = fieldInfo;
}
- public void addField(Fieldable field) {
+ public void addField(IndexableField field) {
if (fieldCount == fields.length) {
int newSize = ArrayUtil.oversize(fieldCount + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- Fieldable[] newArray = new Fieldable[newSize];
+ IndexableField[] newArray = new IndexableField[newSize];
System.arraycopy(fields, 0, newArray, 0, fieldCount);
fields = newArray;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java Sat Aug 27 13:27:01 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -61,27 +60,32 @@ final class DocInverterPerField extends
}
@Override
- public void processFields(final Fieldable[] fields,
+ public void processFields(final IndexableField[] fields,
final int count) throws IOException {
- fieldState.reset(docState.doc.getBoost());
+ fieldState.reset();
final boolean doInvert = consumer.start(fields, count);
for(int i=0;i<count;i++) {
- final Fieldable field = fields[i];
+ final IndexableField field = fields[i];
// TODO FI: this should be "genericized" to querying
// consumer if it wants to see this particular field
// tokenized.
- if (field.isIndexed() && doInvert) {
+ if (field.indexed() && doInvert) {
if (i > 0)
fieldState.position += docState.analyzer == null ? 0 : docState.analyzer.getPositionIncrementGap(fieldInfo.name);
- if (!field.isTokenized()) { // un-tokenized field
- String stringValue = field.stringValue();
+ // TODO (LUCENE-2309): this analysis logic should be
+ // outside of indexer -- field should simply give us
+ // a TokenStream, even for multi-valued fields
+
+ if (!field.tokenized()) { // un-tokenized field
+ final String stringValue = field.stringValue();
+ assert stringValue != null;
final int valueLength = stringValue.length();
parent.singleToken.reinit(stringValue, 0, valueLength);
fieldState.attributeSource = parent.singleToken;
@@ -103,17 +107,17 @@ final class DocInverterPerField extends
final TokenStream stream;
final TokenStream streamValue = field.tokenStreamValue();
- if (streamValue != null)
+ if (streamValue != null) {
stream = streamValue;
- else {
+ } else {
// the field does not have a TokenStream,
// so we have to obtain one from the analyzer
final Reader reader; // find or make Reader
final Reader readerValue = field.readerValue();
- if (readerValue != null)
+ if (readerValue != null) {
reader = readerValue;
- else {
+ } else {
String stringValue = field.stringValue();
if (stringValue == null) {
throw new IllegalArgumentException("field must have either TokenStream, String or Reader value");
@@ -189,7 +193,7 @@ final class DocInverterPerField extends
}
fieldState.offset += docState.analyzer == null ? 0 : docState.analyzer.getOffsetGap(field);
- fieldState.boost *= field.getBoost();
+ fieldState.boost *= field.boost();
}
// LUCENE-2387: don't hang onto the field, so GC can
Added: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java?rev=1162347&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java (added)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentStoredFieldVisitor.java Sat Aug 27 13:27:01 2011
@@ -0,0 +1,142 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.HashSet;
+
+import org.apache.lucene.document.BinaryField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.store.IndexInput;
+
+/** A {@link StoredFieldVisitor} that creates a {@link
+ * Document} containing all stored fields, or only specific
+ * requested fields provided to {@link #DocumentStoredFieldVisitor(Set)}
+ * This is used by {@link IndexReader#document(int)} to load a
+ * document.
+ *
+ * @lucene.experimental */
+
+public class DocumentStoredFieldVisitor extends StoredFieldVisitor {
+ private final Document doc = new Document();
+ private final Set<String> fieldsToAdd;
+
+ /** Load only fields named in the provided <code>Set<String></code>. */
+ public DocumentStoredFieldVisitor(Set<String> fieldsToAdd) {
+ this.fieldsToAdd = fieldsToAdd;
+ }
+
+ /** Load only fields named in the provided <code>Set<String></code>. */
+ public DocumentStoredFieldVisitor(String... fields) {
+ fieldsToAdd = new HashSet<String>(fields.length);
+ for(String field : fields) {
+ fieldsToAdd.add(field);
+ }
+ }
+
+ /** Load all stored fields. */
+ public DocumentStoredFieldVisitor() {
+ this.fieldsToAdd = null;
+ }
+
+ @Override
+ public boolean binaryField(FieldInfo fieldInfo, IndexInput in, int numBytes) throws IOException {
+ if (accept(fieldInfo)) {
+ final byte[] b = new byte[numBytes];
+ in.readBytes(b, 0, b.length);
+ doc.add(new BinaryField(fieldInfo.name, b));
+ } else {
+ in.seek(in.getFilePointer() + numBytes);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean stringField(FieldInfo fieldInfo, IndexInput in, int numUTF8Bytes) throws IOException {
+ if (accept(fieldInfo)) {
+ final byte[] b = new byte[numUTF8Bytes];
+ in.readBytes(b, 0, b.length);
+ FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(fieldInfo.storeTermVector);
+ ft.setStoreTermVectorPositions(fieldInfo.storePositionWithTermVector);
+ ft.setStoreTermVectorOffsets(fieldInfo.storeOffsetWithTermVector);
+ ft.setStoreTermVectors(fieldInfo.storeTermVector);
+ ft.setOmitNorms(fieldInfo.omitNorms);
+ ft.setIndexOptions(fieldInfo.indexOptions);
+ doc.add(new Field(fieldInfo.name,
+ ft,
+ new String(b, "UTF-8")));
+ } else {
+ in.seek(in.getFilePointer() + numUTF8Bytes);
+ }
+ return false;
+ }
+
+ @Override
+ public boolean intField(FieldInfo fieldInfo, int value) {
+ if (accept(fieldInfo)) {
+ FieldType ft = new FieldType(NumericField.TYPE_STORED);
+ ft.setIndexed(fieldInfo.isIndexed);
+ doc.add(new NumericField(fieldInfo.name, ft).setIntValue(value));
+ }
+ return false;
+ }
+
+ @Override
+ public boolean longField(FieldInfo fieldInfo, long value) {
+ if (accept(fieldInfo)) {
+ FieldType ft = new FieldType(NumericField.TYPE_STORED);
+ ft.setIndexed(fieldInfo.isIndexed);
+ doc.add(new NumericField(fieldInfo.name, ft).setLongValue(value));
+ }
+ return false;
+ }
+
+ @Override
+ public boolean floatField(FieldInfo fieldInfo, float value) {
+ if (accept(fieldInfo)) {
+ FieldType ft = new FieldType(NumericField.TYPE_STORED);
+ ft.setIndexed(fieldInfo.isIndexed);
+ doc.add(new NumericField(fieldInfo.name, ft).setFloatValue(value));
+ }
+ return false;
+ }
+
+ @Override
+ public boolean doubleField(FieldInfo fieldInfo, double value) {
+ if (accept(fieldInfo)) {
+ FieldType ft = new FieldType(NumericField.TYPE_STORED);
+ ft.setIndexed(fieldInfo.isIndexed);
+ doc.add(new NumericField(fieldInfo.name, ft).setDoubleValue(value));
+ }
+ return false;
+ }
+
+ private boolean accept(FieldInfo fieldInfo) {
+ return fieldsToAdd == null || fieldsToAdd.contains(fieldInfo.name);
+ }
+
+ public Document getDocument() {
+ return doc;
+ }
+}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Sat Aug 27 13:27:01 2011
@@ -27,7 +27,6 @@ import java.util.Queue;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
@@ -320,7 +319,7 @@ final class DocumentsWriter {
return maybeMerge;
}
- boolean updateDocuments(final Iterable<Document> docs, final Analyzer analyzer,
+ boolean updateDocuments(final Iterable<? extends Iterable<? extends IndexableField>> docs, final Analyzer analyzer,
final Term delTerm) throws CorruptIndexException, IOException {
boolean maybeMerge = preUpdate();
@@ -351,7 +350,7 @@ final class DocumentsWriter {
return postUpdate(flushingDWPT, maybeMerge);
}
- boolean updateDocument(final Document doc, final Analyzer analyzer,
+ boolean updateDocument(final Iterable<? extends IndexableField> doc, final Analyzer analyzer,
final Term delTerm) throws CorruptIndexException, IOException {
boolean maybeMerge = preUpdate();
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Sat Aug 27 13:27:01 2011
@@ -26,7 +26,6 @@ import java.text.NumberFormat;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.store.Directory;
@@ -90,7 +89,7 @@ public class DocumentsWriterPerThread {
PrintStream infoStream;
SimilarityProvider similarityProvider;
int docID;
- Document doc;
+ Iterable<? extends IndexableField> doc;
String maxTermPrefix;
DocState(DocumentsWriterPerThread docWriter) {
@@ -213,7 +212,7 @@ public class DocumentsWriterPerThread {
return retval;
}
- public void updateDocument(Document doc, Analyzer analyzer, Term delTerm) throws IOException {
+ public void updateDocument(Iterable<? extends IndexableField> doc, Analyzer analyzer, Term delTerm) throws IOException {
assert writer.testPoint("DocumentsWriterPerThread addDocument start");
assert deleteQueue != null;
docState.doc = doc;
@@ -263,7 +262,7 @@ public class DocumentsWriterPerThread {
finishDocument(delTerm);
}
- public int updateDocuments(Iterable<Document> docs, Analyzer analyzer, Term delTerm) throws IOException {
+ public int updateDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer, Term delTerm) throws IOException {
assert writer.testPoint("DocumentsWriterPerThread addDocuments start");
assert deleteQueue != null;
docState.analyzer = analyzer;
@@ -280,7 +279,7 @@ public class DocumentsWriterPerThread {
}
int docCount = 0;
try {
- for(Document doc : docs) {
+ for(Iterable<? extends IndexableField> doc : docs) {
docState.doc = doc;
docState.docID = numDocsInRAM;
docCount++;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java Sat Aug 27 13:27:01 2011
@@ -30,9 +30,9 @@ public final class FieldInfo {
// true if term vector for this field should be stored
- boolean storeTermVector;
- boolean storeOffsetWithTermVector;
- boolean storePositionWithTermVector;
+ public boolean storeTermVector;
+ public boolean storeOffsetWithTermVector;
+ public boolean storePositionWithTermVector;
public boolean omitNorms; // omit norms associated with indexed fields
public IndexOptions indexOptions;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java Sat Aug 27 13:27:01 2011
@@ -39,8 +39,8 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.CodecUtil;
-/** Access to the Fieldable Info file that describes document fields and whether or
- * not they are indexed. Each segment has a separate Fieldable Info file. Objects
+/** Access to the Field Info file that describes document fields and whether or
+ * not they are indexed. Each segment has a separate Field Info file. Objects
* of this class are thread-safe for multiple readers, but only one thread can
* be adding documents at a time, with no other reader or writer threads
* accessing this object.
@@ -381,7 +381,7 @@ public final class FieldInfos implements
/**
* Calls 5 parameter add with false for all TermVector parameters.
*
- * @param name The name of the Fieldable
+ * @param name The name of the IndexableField
* @param isIndexed true if the field is indexed
* @see #addOrUpdate(String, boolean, boolean, boolean, boolean)
*/
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInvertState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInvertState.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInvertState.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInvertState.java Sat Aug 27 13:27:01 2011
@@ -50,14 +50,14 @@ public final class FieldInvertState {
* Re-initialize the state, using this boost value.
* @param docBoost boost value to use.
*/
- void reset(float docBoost) {
+ void reset() {
position = 0;
length = 0;
numOverlap = 0;
offset = 0;
maxTermFrequency = 0;
uniqueTermCount = 0;
- boost = docBoost;
+ boost = 1.0f;
attributeSource = null;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsReader.java Sat Aug 27 13:27:01 2011
@@ -17,16 +17,9 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.AbstractField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.NumericField;
+import java.io.IOException;
+
import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -34,9 +27,6 @@ import org.apache.lucene.util.CloseableT
import org.apache.lucene.util.IOUtils;
import java.io.Closeable;
-import java.io.IOException;
-import java.io.Reader;
-import java.util.ArrayList;
/**
* Class responsible for access to stored document fields.
@@ -49,7 +39,8 @@ public final class FieldsReader implemen
private final static int FORMAT_SIZE = 4;
private final FieldInfos fieldInfos;
-
+ private CloseableThreadLocal<IndexInput> fieldsStreamTL = new CloseableThreadLocal<IndexInput>();
+
// The main fieldStream, used only for cloning.
private final IndexInput cloneableFieldsStream;
@@ -68,7 +59,6 @@ public final class FieldsReader implemen
// file. This will be 0 if we have our own private file.
private int docStoreOffset;
- private CloseableThreadLocal<IndexInput> fieldsStreamTL = new CloseableThreadLocal<IndexInput>();
private boolean isOriginal = false;
/** Returns a cloned FieldsReader that shares open
@@ -200,50 +190,52 @@ public final class FieldsReader implemen
indexStream.seek(FORMAT_SIZE + (docID + docStoreOffset) * 8L);
}
- public final Document doc(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+ public final void visitDocument(int n, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
seekIndex(n);
- long position = indexStream.readLong();
- fieldsStream.seek(position);
+ fieldsStream.seek(indexStream.readLong());
- Document doc = new Document();
- int numFields = fieldsStream.readVInt();
- out: for (int i = 0; i < numFields; i++) {
+ final int numFields = fieldsStream.readVInt();
+ for (int fieldIDX = 0; fieldIDX < numFields; fieldIDX++) {
int fieldNumber = fieldsStream.readVInt();
- FieldInfo fi = fieldInfos.fieldInfo(fieldNumber);
- FieldSelectorResult acceptField = fieldSelector == null ? FieldSelectorResult.LOAD : fieldSelector.accept(fi.name);
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
int bits = fieldsStream.readByte() & 0xFF;
- assert bits <= (FieldsWriter.FIELD_IS_NUMERIC_MASK | FieldsWriter.FIELD_IS_TOKENIZED | FieldsWriter.FIELD_IS_BINARY): "bits=" + Integer.toHexString(bits);
+ assert bits <= (FieldsWriter.FIELD_IS_NUMERIC_MASK | FieldsWriter.FIELD_IS_BINARY): "bits=" + Integer.toHexString(bits);
- boolean tokenize = (bits & FieldsWriter.FIELD_IS_TOKENIZED) != 0;
- boolean binary = (bits & FieldsWriter.FIELD_IS_BINARY) != 0;
+ final boolean binary = (bits & FieldsWriter.FIELD_IS_BINARY) != 0;
final int numeric = bits & FieldsWriter.FIELD_IS_NUMERIC_MASK;
- switch (acceptField) {
- case LOAD:
- addField(doc, fi, binary, tokenize, numeric);
+ final boolean doStop;
+ if (binary) {
+ final int numBytes = fieldsStream.readVInt();
+ doStop = visitor.binaryField(fieldInfo, fieldsStream, numBytes);
+ } else if (numeric != 0) {
+ switch(numeric) {
+ case FieldsWriter.FIELD_IS_NUMERIC_INT:
+ doStop = visitor.intField(fieldInfo, fieldsStream.readInt());
break;
- case LOAD_AND_BREAK:
- addField(doc, fi, binary, tokenize, numeric);
- break out; //Get out of this loop
- case LAZY_LOAD:
- addFieldLazy(doc, fi, binary, tokenize, true, numeric);
+ case FieldsWriter.FIELD_IS_NUMERIC_LONG:
+ doStop = visitor.longField(fieldInfo, fieldsStream.readLong());
break;
- case LATENT:
- addFieldLazy(doc, fi, binary, tokenize, false, numeric);
+ case FieldsWriter.FIELD_IS_NUMERIC_FLOAT:
+ doStop = visitor.floatField(fieldInfo, Float.intBitsToFloat(fieldsStream.readInt()));
break;
- case SIZE:
- skipFieldBytes(addFieldSize(doc, fi, binary, numeric));
+ case FieldsWriter.FIELD_IS_NUMERIC_DOUBLE:
+ doStop = visitor.doubleField(fieldInfo, Double.longBitsToDouble(fieldsStream.readLong()));
break;
- case SIZE_AND_BREAK:
- addFieldSize(doc, fi, binary, numeric);
- break out; //Get out of this loop
default:
- skipField(numeric);
+ throw new FieldReaderException("Invalid numeric type: " + Integer.toHexString(numeric));
+ }
+ } else {
+ // Text:
+ final int numUTF8Bytes = fieldsStream.readVInt();
+ doStop = visitor.stringField(fieldInfo, fieldsStream, numUTF8Bytes);
}
- }
- return doc;
+ if (doStop) {
+ return;
+ }
+ }
}
/** Returns the length in bytes of each raw document in a
@@ -300,225 +292,4 @@ public final class FieldsReader implemen
private void skipFieldBytes(int toRead) throws IOException {
fieldsStream.seek(fieldsStream.getFilePointer() + toRead);
}
-
- private NumericField loadNumericField(FieldInfo fi, int numeric) throws IOException {
- assert numeric != 0;
- switch(numeric) {
- case FieldsWriter.FIELD_IS_NUMERIC_INT:
- return new NumericField(fi.name, Field.Store.YES, fi.isIndexed).setIntValue(fieldsStream.readInt());
- case FieldsWriter.FIELD_IS_NUMERIC_LONG:
- return new NumericField(fi.name, Field.Store.YES, fi.isIndexed).setLongValue(fieldsStream.readLong());
- case FieldsWriter.FIELD_IS_NUMERIC_FLOAT:
- return new NumericField(fi.name, Field.Store.YES, fi.isIndexed).setFloatValue(Float.intBitsToFloat(fieldsStream.readInt()));
- case FieldsWriter.FIELD_IS_NUMERIC_DOUBLE:
- return new NumericField(fi.name, Field.Store.YES, fi.isIndexed).setDoubleValue(Double.longBitsToDouble(fieldsStream.readLong()));
- default:
- throw new FieldReaderException("Invalid numeric type: " + Integer.toHexString(numeric));
- }
- }
-
- private void addFieldLazy(Document doc, FieldInfo fi, boolean binary, boolean tokenize, boolean cacheResult, int numeric) throws IOException {
- final AbstractField f;
- if (binary) {
- int toRead = fieldsStream.readVInt();
- long pointer = fieldsStream.getFilePointer();
- f = new LazyField(fi.name, Field.Store.YES, toRead, pointer, binary, cacheResult);
- //Need to move the pointer ahead by toRead positions
- fieldsStream.seek(pointer + toRead);
- } else if (numeric != 0) {
- f = loadNumericField(fi, numeric);
- } else {
- Field.Store store = Field.Store.YES;
- Field.Index index = Field.Index.toIndex(fi.isIndexed, tokenize);
- Field.TermVector termVector = Field.TermVector.toTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
-
- int length = fieldsStream.readVInt();
- long pointer = fieldsStream.getFilePointer();
- //Skip ahead of where we are by the length of what is stored
- fieldsStream.seek(pointer+length);
- f = new LazyField(fi.name, store, index, termVector, length, pointer, binary, cacheResult);
- }
-
- f.setOmitNorms(fi.omitNorms);
- f.setIndexOptions(fi.indexOptions);
- doc.add(f);
- }
-
- private void addField(Document doc, FieldInfo fi, boolean binary, boolean tokenize, int numeric) throws CorruptIndexException, IOException {
- final AbstractField f;
-
- if (binary) {
- int toRead = fieldsStream.readVInt();
- final byte[] b = new byte[toRead];
- fieldsStream.readBytes(b, 0, b.length);
- f = new Field(fi.name, b);
- } else if (numeric != 0) {
- f = loadNumericField(fi, numeric);
- } else {
- Field.Index index = Field.Index.toIndex(fi.isIndexed, tokenize);
- Field.TermVector termVector = Field.TermVector.toTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
- f = new Field(fi.name, // name
- fieldsStream.readString(), // read value
- Field.Store.YES,
- index,
- termVector);
- }
-
- f.setIndexOptions(fi.indexOptions);
- f.setOmitNorms(fi.omitNorms);
- doc.add(f);
- }
-
- // Add the size of field as a byte[] containing the 4 bytes of the integer byte size (high order byte first; char = 2 bytes)
- // Read just the size -- caller must skip the field content to continue reading fields
- // Return the size in bytes or chars, depending on field type
- private int addFieldSize(Document doc, FieldInfo fi, boolean binary, int numeric) throws IOException {
- final int bytesize, size;
- switch(numeric) {
- case 0:
- size = fieldsStream.readVInt();
- bytesize = binary ? size : 2*size;
- break;
- case FieldsWriter.FIELD_IS_NUMERIC_INT:
- case FieldsWriter.FIELD_IS_NUMERIC_FLOAT:
- size = bytesize = 4;
- break;
- case FieldsWriter.FIELD_IS_NUMERIC_LONG:
- case FieldsWriter.FIELD_IS_NUMERIC_DOUBLE:
- size = bytesize = 8;
- break;
- default:
- throw new FieldReaderException("Invalid numeric type: " + Integer.toHexString(numeric));
- }
- byte[] sizebytes = new byte[4];
- sizebytes[0] = (byte) (bytesize>>>24);
- sizebytes[1] = (byte) (bytesize>>>16);
- sizebytes[2] = (byte) (bytesize>>> 8);
- sizebytes[3] = (byte) bytesize ;
- doc.add(new Field(fi.name, sizebytes));
- return size;
- }
-
- /**
- * A Lazy implementation of Fieldable that defers loading of fields until asked for, instead of when the Document is
- * loaded.
- */
- private class LazyField extends AbstractField implements Fieldable {
- private int toRead;
- private long pointer;
- private final boolean cacheResult;
-
- public LazyField(String name, Field.Store store, int toRead, long pointer, boolean isBinary, boolean cacheResult) {
- super(name, store, Field.Index.NO, Field.TermVector.NO);
- this.toRead = toRead;
- this.pointer = pointer;
- this.isBinary = isBinary;
- this.cacheResult = cacheResult;
- if (isBinary)
- binaryLength = toRead;
- lazy = true;
- }
-
- public LazyField(String name, Field.Store store, Field.Index index, Field.TermVector termVector, int toRead, long pointer, boolean isBinary, boolean cacheResult) {
- super(name, store, index, termVector);
- this.toRead = toRead;
- this.pointer = pointer;
- this.isBinary = isBinary;
- this.cacheResult = cacheResult;
- if (isBinary)
- binaryLength = toRead;
- lazy = true;
- }
-
- private IndexInput getFieldStream() {
- IndexInput localFieldsStream = fieldsStreamTL.get();
- if (localFieldsStream == null) {
- localFieldsStream = (IndexInput) cloneableFieldsStream.clone();
- fieldsStreamTL.set(localFieldsStream);
- }
- return localFieldsStream;
- }
-
- /** The value of the field as a Reader, or null. If null, the String value,
- * binary value, or TokenStream value is used. Exactly one of stringValue(),
- * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
- public Reader readerValue() {
- ensureOpen();
- return null;
- }
-
- /** The value of the field as a TokenStream, or null. If null, the Reader value,
- * String value, or binary value is used. Exactly one of stringValue(),
- * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
- public TokenStream tokenStreamValue() {
- ensureOpen();
- return null;
- }
-
- /** The value of the field as a String, or null. If null, the Reader value,
- * binary value, or TokenStream value is used. Exactly one of stringValue(),
- * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
- public String stringValue() {
- ensureOpen();
- if (isBinary)
- return null;
- else {
- if (fieldsData == null) {
- String result = null;
- IndexInput localFieldsStream = getFieldStream();
- try {
- localFieldsStream.seek(pointer);
- byte[] bytes = new byte[toRead];
- localFieldsStream.readBytes(bytes, 0, toRead);
- result = new String(bytes, "UTF-8");
- } catch (IOException e) {
- throw new FieldReaderException(e);
- }
- if (cacheResult == true){
- fieldsData = result;
- }
- return result;
- } else {
- return (String) fieldsData;
- }
- }
- }
-
- @Override
- public byte[] getBinaryValue(byte[] result) {
- ensureOpen();
-
- if (isBinary) {
- if (fieldsData == null) {
- // Allocate new buffer if result is null or too small
- final byte[] b;
- if (result == null || result.length < toRead)
- b = new byte[toRead];
- else
- b = result;
-
- IndexInput localFieldsStream = getFieldStream();
-
- // Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people
- // since they are already handling this exception when getting the document
- try {
- localFieldsStream.seek(pointer);
- localFieldsStream.readBytes(b, 0, toRead);
- } catch (IOException e) {
- throw new FieldReaderException(e);
- }
-
- binaryOffset = 0;
- binaryLength = toRead;
- if (cacheResult == true){
- fieldsData = b;
- }
- return b;
- } else {
- return (byte[]) fieldsData;
- }
- } else
- return null;
- }
- }
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldsWriter.java Sat Aug 27 13:27:01 2011
@@ -17,19 +17,16 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.List;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.NumericField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
final class FieldsWriter {
- static final int FIELD_IS_TOKENIZED = 1 << 0;
+ // NOTE: bit 0 is free here! You can steal it!
static final int FIELD_IS_BINARY = 1 << 1;
// the old bit 1 << 2 was compressed, is now left out
@@ -138,15 +135,17 @@ final class FieldsWriter {
}
}
- final void writeField(int fieldNumber, Fieldable field) throws IOException {
+ final void writeField(int fieldNumber, IndexableField field) throws IOException {
fieldsStream.writeVInt(fieldNumber);
int bits = 0;
- if (field.isTokenized())
- bits |= FIELD_IS_TOKENIZED;
- if (field.isBinary())
- bits |= FIELD_IS_BINARY;
- if (field instanceof NumericField) {
- switch (((NumericField) field).getDataType()) {
+ final BytesRef bytes;
+ final String string;
+ // TODO: maybe a field should serialize itself?
+ // this way we don't bake into indexer all these
+ // specific encodings for different fields? and apps
+ // can customize...
+ if (field.numeric()) {
+ switch (field.numericDataType()) {
case INT:
bits |= FIELD_IS_NUMERIC_INT; break;
case LONG:
@@ -158,23 +157,31 @@ final class FieldsWriter {
default:
assert false : "Should never get here";
}
+ string = null;
+ bytes = null;
+ } else {
+ bytes = field.binaryValue();
+ if (bytes != null) {
+ bits |= FIELD_IS_BINARY;
+ string = null;
+ } else {
+ string = field.stringValue();
+ }
}
+
fieldsStream.writeByte((byte) bits);
- if (field.isBinary()) {
- final byte[] data;
- final int len;
- final int offset;
- data = field.getBinaryValue();
- len = field.getBinaryLength();
- offset = field.getBinaryOffset();
-
- fieldsStream.writeVInt(len);
- fieldsStream.writeBytes(data, offset, len);
- } else if (field instanceof NumericField) {
- final NumericField nf = (NumericField) field;
- final Number n = nf.getNumericValue();
- switch (nf.getDataType()) {
+ if (bytes != null) {
+ fieldsStream.writeVInt(bytes.length);
+ fieldsStream.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ } else if (string != null) {
+ fieldsStream.writeString(field.stringValue());
+ } else {
+ final Number n = field.numericValue();
+ if (n == null) {
+ throw new IllegalArgumentException("field " + field.name() + " is stored but does not have binaryValue, stringValue nor numericValue");
+ }
+ switch (field.numericDataType()) {
case INT:
fieldsStream.writeInt(n.intValue()); break;
case LONG:
@@ -186,8 +193,6 @@ final class FieldsWriter {
default:
assert false : "Should never get here";
}
- } else {
- fieldsStream.writeString(field.stringValue());
}
}
@@ -207,21 +212,21 @@ final class FieldsWriter {
assert fieldsStream.getFilePointer() == position;
}
- final void addDocument(Document doc, FieldInfos fieldInfos) throws IOException {
+ final void addDocument(Iterable<? extends IndexableField> doc, FieldInfos fieldInfos) throws IOException {
indexStream.writeLong(fieldsStream.getFilePointer());
int storedCount = 0;
- List<Fieldable> fields = doc.getFields();
- for (Fieldable field : fields) {
- if (field.isStored())
- storedCount++;
+ for (IndexableField field : doc) {
+ if (field.stored()) {
+ storedCount++;
+ }
}
fieldsStream.writeVInt(storedCount);
-
- for (Fieldable field : fields) {
- if (field.isStored())
+ for (IndexableField field : doc) {
+ if (field.stored()) {
writeField(fieldInfos.fieldNumber(field.name()), field);
+ }
}
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Sat Aug 27 13:27:01 2011
@@ -17,9 +17,8 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -351,9 +350,9 @@ public class FilterIndexReader extends I
}
@Override
- public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+ public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
ensureOpen();
- return in.document(n, fieldSelector);
+ in.document(docID, visitor);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1162347&r1=1162346&r2=1162347&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Sat Aug 27 13:27:01 2011
@@ -22,7 +22,6 @@ import java.util.Comparator;
import java.util.Map;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.PostingsConsumer;
@@ -82,15 +81,17 @@ final class FreqProxTermsWriterPerField
}
@Override
- boolean start(Fieldable[] fields, int count) {
- for(int i=0;i<count;i++)
- if (fields[i].isIndexed())
+ boolean start(IndexableField[] fields, int count) {
+ for(int i=0;i<count;i++) {
+ if (fields[i].indexed()) {
return true;
+ }
+ }
return false;
}
@Override
- void start(Fieldable f) {
+ void start(IndexableField f) {
if (fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
} else {