You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/06/09 20:00:55 UTC
svn commit: r1134021 [1/2] - in /lucene/dev/branches/fieldtype: ./
lucene/src/java/org/apache/lucene/analysis/
lucene/src/java/org/apache/lucene/document/
lucene/src/java/org/apache/lucene/index/
lucene/src/java/org/apache/lucene/search/ lucene/src/tes...
Author: mikemccand
Date: Thu Jun 9 18:00:54 2011
New Revision: 1134021
URL: http://svn.apache.org/viewvc?rev=1134021&view=rev
Log:
LUCENE-3177: initial cut, decoupling indexer from concrete doc/field impl
Added:
lucene/dev/branches/fieldtype/TODO
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexableField.java (with props)
Modified:
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/Analyzer.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/package.html
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/AbstractField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Document.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Field.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Fieldable.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/NumericField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInvertState.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsReader.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsWriter.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/search/Similarity.java
lucene/dev/branches/fieldtype/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestDocBoost.java
lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java
lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java
lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/BinaryField.java
lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/TrieField.java
lucene/dev/branches/fieldtype/solr/src/test/org/apache/solr/schema/PolyFieldTest.java
Added: lucene/dev/branches/fieldtype/TODO
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/TODO?rev=1134021&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/TODO (added)
+++ lucene/dev/branches/fieldtype/TODO Thu Jun 9 18:00:54 2011
@@ -0,0 +1,7 @@
+Issues:
+
+ - LUCENE-3177 -- decouple indexer from doc/field impls
+ - LUCENE-2308 -- FieldType hierarhcy
+ - LUCENE-2309 -- decouple indexer from analysis
+
+maybe also need extensiblity for a Field to save itself into stored fields; this would shrink IndexableField API
\ No newline at end of file
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/Analyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/Analyzer.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/Analyzer.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/Analyzer.java Thu Jun 9 18:00:54 2011
@@ -22,11 +22,10 @@ import java.io.IOException;
import java.io.Closeable;
import java.lang.reflect.Modifier;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.CloseableThreadLocal;
import org.apache.lucene.store.AlreadyClosedException;
-import org.apache.lucene.document.Fieldable;
-
/** An Analyzer builds TokenStreams, which analyze text. It thus represents a
* policy for extracting index terms from text.
* <p>
@@ -111,16 +110,16 @@ public abstract class Analyzer implement
}
/**
- * Invoked before indexing a Fieldable instance if
+ * Invoked before indexing a IndexableField instance if
* terms have already been added to that field. This allows custom
* analyzers to place an automatic position increment gap between
- * Fieldable instances using the same field name. The default value
+ * IndexbleField instances using the same field name. The default value
* position increment gap is 0. With a 0 position increment gap and
* the typical default token position increment of 1, all terms in a field,
- * including across Fieldable instances, are in successive positions, allowing
- * exact PhraseQuery matches, for instance, across Fieldable instance boundaries.
+ * including across IndexableField instances, are in successive positions, allowing
+ * exact PhraseQuery matches, for instance, across IndexableField instance boundaries.
*
- * @param fieldName Fieldable name being indexed.
+ * @param fieldName IndexableField name being indexed.
* @return position increment gap, added to the next token emitted from {@link #tokenStream(String,Reader)}
*/
public int getPositionIncrementGap(String fieldName) {
@@ -138,11 +137,13 @@ public abstract class Analyzer implement
* @param field the field just indexed
* @return offset gap, added to the next token emitted from {@link #tokenStream(String,Reader)}
*/
- public int getOffsetGap(Fieldable field) {
- if (field.isTokenized())
+ // nocommit cut to IndexableField
+ public int getOffsetGap(IndexableField field) {
+ if (field.isTokenized()) {
return 1;
- else
+ } else {
return 0;
+ }
}
/** Frees persistent resources used by this Analyzer */
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/package.html?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/package.html (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/analysis/package.html Thu Jun 9 18:00:54 2011
@@ -120,7 +120,7 @@ There are many post tokenization steps t
Applications usually do not invoke analysis – Lucene does it for them:
<ul>
<li>At indexing, as a consequence of
- {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document) addDocument(doc)},
+ {@link org.apache.lucene.index.IndexWriter#addDocument(Iterable) addDocument(doc)},
the Analyzer in effect for indexing is invoked for each indexed field of the added document.
</li>
<li>At search, as a consequence of
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/AbstractField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/AbstractField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/AbstractField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/AbstractField.java Thu Jun 9 18:00:54 2011
@@ -15,10 +15,11 @@ package org.apache.lucene.document;
* limitations under the License.
*/
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.FieldInvertState; // for javadocs
import org.apache.lucene.search.PhraseQuery; // for javadocs
import org.apache.lucene.search.spans.SpanQuery; // for javadocs
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.StringHelper; // for javadocs
@@ -156,35 +157,50 @@ public abstract class AbstractField impl
}
/** True iff the value of the filed is stored as binary */
- public final boolean isBinary() {
+ public final boolean isBinary() {
return isBinary;
}
- /**
- * Return the raw byte[] for the binary field. Note that
- * you must also call {@link #getBinaryLength} and {@link
- * #getBinaryOffset} to know which range of bytes in this
- * returned array belong to the field.
- * @return reference to the Field value as byte[].
- */
- public byte[] getBinaryValue() {
+ private byte[] getBinaryValue() {
return getBinaryValue(null);
}
- public byte[] getBinaryValue(byte[] result){
+ private byte[] getBinaryValue(byte[] result /* unused */){
if (isBinary || fieldsData instanceof byte[])
return (byte[]) fieldsData;
else
return null;
}
+ public boolean isNumeric() {
+ return false;
+ }
+
+ public BytesRef binaryValue(BytesRef reuse) {
+ final byte[] bytes = getBinaryValue();
+ if (bytes != null) {
+ if (reuse == null) {
+ return new BytesRef(bytes,
+ getBinaryOffset(),
+ getBinaryLength());
+ } else {
+ reuse.bytes = bytes;
+ reuse.offset = getBinaryOffset();
+ reuse.length = getBinaryLength();
+ return reuse;
+ }
+ } else {
+ return null;
+ }
+ }
+
/**
* Returns length of byte[] segment that is used as value, if Field is not binary
* returned value is undefined
* @return length of byte[] segment that represents this Field value
*/
- public int getBinaryLength() {
+ private int getBinaryLength() {
if (isBinary) {
return binaryLength;
} else if (fieldsData instanceof byte[])
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Document.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Document.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Document.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Document.java Thu Jun 9 18:00:54 2011
@@ -17,7 +17,7 @@ package org.apache.lucene.document;
* limitations under the License.
*/
-import java.util.*; // for javadoc
+import java.util.*;
import org.apache.lucene.search.IndexSearcher; // for javadoc
import org.apache.lucene.search.ScoreDoc; // for javadoc
import org.apache.lucene.index.IndexReader; // for javadoc
@@ -35,13 +35,20 @@ import org.apache.lucene.index.IndexRead
* ScoreDoc#doc} or {@link IndexReader#document(int)}.
*/
-public final class Document {
+public final class Document implements Iterable<Fieldable> {
+
List<Fieldable> fields = new ArrayList<Fieldable>();
private float boost = 1.0f;
/** Constructs a new document with no fields. */
public Document() {}
+ // @Override not until Java 1.6
+ public Iterator<Fieldable> iterator() {
+ // nocommit -- must multiply in docBoost to each
+ // provided field
+ return fields.iterator();
+ }
/** Sets a boost factor for hits on any field of this document. This value
* will be multiplied into the score of all hits on this document.
@@ -70,6 +77,7 @@ public final class Document {
*
* @see #setBoost(float)
*/
+ // @Override not until Java 1.6
public float getBoost() {
return boost;
}
@@ -275,7 +283,7 @@ public final class Document {
List<byte[]> result = new ArrayList<byte[]>();
for (Fieldable field : fields) {
if (field.name().equals(name) && (field.isBinary()))
- result.add(field.getBinaryValue());
+ result.add(field.binaryValue(null).bytes);
}
if (result.size() == 0)
@@ -296,7 +304,7 @@ public final class Document {
public final byte[] getBinaryValue(String name) {
for (Fieldable field : fields) {
if (field.name().equals(name) && (field.isBinary()))
- return field.getBinaryValue();
+ return field.binaryValue(null).bytes;
}
return null;
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Field.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Field.java Thu Jun 9 18:00:54 2011
@@ -274,7 +274,14 @@ public final class Field extends Abstrac
/** The TokesStream for this field to be used when indexing, or null. If null, the Reader value
* or String value is analyzed to produce the indexed tokens. */
public TokenStream tokenStreamValue() { return tokenStream; }
-
+
+ public Number getNumericValue() {
+ return null;
+ }
+
+ public NumericField.DataType getDataType() {
+ return null;
+ }
/** <p>Expert: change the value of this field. This can
* be used during indexing to re-use a single Field
@@ -426,7 +433,7 @@ public final class Field extends Abstrac
/**
* Create a tokenized and indexed field that is not stored. Term vectors will
* not be stored. The Reader is read only when the Document is added to the index,
- * i.e. you may not close the Reader until {@link IndexWriter#addDocument(Document)}
+ * i.e. you may not close the Reader until {@link IndexWriter#addDocument(Iterable)}
* has been called.
*
* @param name The name of the field
@@ -440,7 +447,7 @@ public final class Field extends Abstrac
/**
* Create a tokenized and indexed field that is not stored, optionally with
* storing term vectors. The Reader is read only when the Document is added to the index,
- * i.e. you may not close the Reader until {@link IndexWriter#addDocument(Document)}
+ * i.e. you may not close the Reader until {@link IndexWriter#addDocument(Iterable)}
* has been called.
*
* @param name The name of the field
@@ -471,7 +478,7 @@ public final class Field extends Abstrac
* Create a tokenized and indexed field that is not stored. Term vectors will
* not be stored. This is useful for pre-analyzed fields.
* The TokenStream is read only when the Document is added to the index,
- * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(Document)}
+ * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(Iterable)}
* has been called.
*
* @param name The name of the field
@@ -486,7 +493,7 @@ public final class Field extends Abstrac
* Create a tokenized and indexed field that is not stored, optionally with
* storing term vectors. This is useful for pre-analyzed fields.
* The TokenStream is read only when the Document is added to the index,
- * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(Document)}
+ * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(Iterable)}
* has been called.
*
* @param name The name of the field
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Fieldable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Fieldable.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Fieldable.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/Fieldable.java Thu Jun 9 18:00:54 2011
@@ -16,12 +16,11 @@ package org.apache.lucene.document;
* limitations under the License.
*/
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.index.FieldInvertState; // for javadocs
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.PhraseQuery; // for javadocs
import org.apache.lucene.search.spans.SpanQuery; // for javadocs
-
-import java.io.Reader;
+import org.apache.lucene.util.BytesRef; // for javadocs
/**
* Synonymous with {@link Field}.
@@ -33,7 +32,8 @@ import java.io.Reader;
* </p>
*
**/
-public interface Fieldable {
+public interface Fieldable extends IndexableField {
+
/** Sets the boost factor hits on this field. This value will be
* multiplied into the score of all hits on this this field of this
* document.
@@ -66,46 +66,46 @@ public interface Fieldable {
*
* @see #setBoost(float)
*/
- float getBoost();
+ //float getBoost();
/** Returns the name of the field as an interned string.
* For example "date", "title", "body", ...
*/
- String name();
+ //String name();
/** The value of the field as a String, or null.
* <p>
* For indexing, if isStored()==true, the stringValue() will be used as the stored field value
- * unless isBinary()==true, in which case getBinaryValue() will be used.
+ * unless isBinary()==true, in which case binaryValue() will be used.
*
* If isIndexed()==true and isTokenized()==false, this String value will be indexed as a single token.
* If isIndexed()==true and isTokenized()==true, then tokenStreamValue() will be used to generate indexed tokens if not null,
* else readerValue() will be used to generate indexed tokens if not null, else stringValue() will be used to generate tokens.
*/
- public String stringValue();
+ //public String stringValue();
/** The value of the field as a Reader, which can be used at index time to generate indexed tokens.
* @see #stringValue()
*/
- public Reader readerValue();
+ //public Reader readerValue();
/** The TokenStream for this field to be used when indexing, or null.
* @see #stringValue()
*/
- public TokenStream tokenStreamValue();
+ //public TokenStream tokenStreamValue();
/** True if the value of the field is to be stored in the index for return
with search hits. */
- boolean isStored();
+ //boolean isStored();
/** True if the value of the field is to be indexed, so that it may be
searched on. */
- boolean isIndexed();
+ //boolean isIndexed();
/** True if the value of the field should be tokenized as text prior to
indexing. Un-tokenized fields are indexed as a single word and may not be
Reader-valued. */
- boolean isTokenized();
+ //boolean isTokenized();
/** True if the term or terms used to index this field are stored as a term
* vector, available from {@link org.apache.lucene.index.IndexReader#getTermFreqVector(int,String)}.
@@ -115,24 +115,24 @@ public interface Fieldable {
*
* @see org.apache.lucene.index.IndexReader#getTermFreqVector(int, String)
*/
- boolean isTermVectorStored();
+ //boolean isTermVectorStored();
/**
* True if terms are stored as term vector together with their offsets
* (start and end positon in source text).
*/
- boolean isStoreOffsetWithTermVector();
+ //boolean isStoreOffsetWithTermVector();
/**
* True if terms are stored as term vector together with their token positions.
*/
- boolean isStorePositionWithTermVector();
+ //boolean isStorePositionWithTermVector();
/** True if the value of the field is stored as binary */
- boolean isBinary();
+ boolean isBinary();
/** True if norms are omitted for this indexed field */
- boolean getOmitNorms();
+ //boolean getOmitNorms();
/** Expert:
*
@@ -143,7 +143,7 @@ public interface Fieldable {
/**
* Indicates whether a Field is Lazy or not. The semantics of Lazy loading are such that if a Field is lazily loaded, retrieving
- * it's values via {@link #stringValue()} or {@link #getBinaryValue()} is only valid as long as the {@link org.apache.lucene.index.IndexReader} that
+ * it's values via {@link #stringValue()} or {@link #binaryValue(BytesRef)} is only valid as long as the {@link org.apache.lucene.index.IndexReader} that
* retrieved the {@link Document} is still open.
*
* @return true if this field can be loaded lazily
@@ -155,14 +155,14 @@ public interface Fieldable {
* returned value is undefined
* @return index of the first character in byte[] segment that represents this Field value
*/
- abstract int getBinaryOffset();
+ //abstract int getBinaryOffset();
/**
* Returns length of byte[] segment that is used as value, if Field is not binary
* returned value is undefined
* @return length of byte[] segment that represents this Field value
*/
- abstract int getBinaryLength();
+ //abstract int getBinaryLength();
/**
* Return the raw byte[] for the binary field. Note that
@@ -171,17 +171,24 @@ public interface Fieldable {
* returned array belong to the field.
* @return reference to the Field value as byte[].
*/
- abstract byte[] getBinaryValue();
+ //abstract byte[] getBinaryValue();
+
+ // nocommit api break
+ //abstract BytesRef binaryValue();
+
+ //abstract NumericField.DataType getDataType();
+
+ //abstract Number getNumericValue();
/**
* Return the raw byte[] for the binary field. Note that
- * you must also call {@link #getBinaryLength} and {@link
- * #getBinaryOffset} to know which range of bytes in this
+ * you must also call {@link #binaryValue}
+ * to know which range of bytes in this
* returned array belong to the field.<p>
* About reuse: if you pass in the result byte[] and it is
* used, likely the underlying implementation will hold
* onto this byte[] and return it in future calls to
- * {@link #getBinaryValue()}.
+ * {@link #binaryValue(BytesRef)}.
* So if you subsequently re-use the same byte[] elsewhere
* it will alter this Fieldable's value.
* @param result User defined buffer that will be used if
@@ -189,10 +196,11 @@ public interface Fieldable {
* buffer is allocated
* @return reference to the Field value as byte[].
*/
- abstract byte[] getBinaryValue(byte[] result);
+ // nocommit -- remove this too; add resuse param to binaryValue
+ //abstract byte[] getBinaryValue(byte[] result);
/** @see #setOmitTermFreqAndPositions */
- boolean getOmitTermFreqAndPositions();
+ //boolean getOmitTermFreqAndPositions();
/** Expert:
*
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/NumericField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/NumericField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/NumericField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/document/NumericField.java Thu Jun 9 18:00:54 2011
@@ -226,12 +226,6 @@ public final class NumericField extends
}
/** Returns always <code>null</code> for numeric fields */
- @Override
- public byte[] getBinaryValue(byte[] result){
- return null;
- }
-
- /** Returns always <code>null</code> for numeric fields */
public Reader readerValue() {
return null;
}
@@ -260,6 +254,10 @@ public final class NumericField extends
public DataType getDataType() {
return type;
}
+
+ public boolean isNumeric() {
+ return true;
+ }
/**
* Initializes the field with the supplied <code>long</code> value.
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java Thu Jun 9 18:00:54 2011
@@ -18,11 +18,10 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
abstract class DocFieldConsumerPerField {
/** Processes all occurrences of a single field */
- abstract void processFields(Fieldable[] fields, int count) throws IOException;
+ abstract void processFields(IndexableField[] fields, int count) throws IOException;
abstract void abort();
abstract FieldInfo getFieldInfo();
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java Thu Jun 9 18:00:54 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
final class DocFieldConsumersPerField extends DocFieldConsumerPerField {
@@ -35,7 +34,7 @@ final class DocFieldConsumersPerField ex
}
@Override
- public void processFields(Fieldable[] fields, int count) throws IOException {
+ public void processFields(IndexableField[] fields, int count) throws IOException {
one.processFields(fields, count);
two.processFields(fields, count);
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Thu Jun 9 18:00:54 2011
@@ -22,11 +22,8 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ArrayUtil;
@@ -181,22 +178,16 @@ final class DocFieldProcessor extends Do
consumer.startDocument();
fieldsWriter.startDocument();
- final Document doc = docState.doc;
-
fieldCount = 0;
final int thisFieldGen = fieldGen++;
- final List<Fieldable> docFields = doc.getFields();
- final int numDocFields = docFields.size();
-
// Absorb any new fields first seen in this document.
// Also absorb any changes to fields we had already
// seen before (eg suddenly turning on norms or
// vectors, etc.):
- for(int i=0;i<numDocFields;i++) {
- Fieldable field = docFields.get(i);
+ for(IndexableField field : docState.doc) {
final String fieldName = field.name();
// Make sure we have a PerField allocated
@@ -214,20 +205,21 @@ final class DocFieldProcessor extends Do
// to have a new "thing" my Fields can do, I can
// easily add it
FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.isIndexed(), field.isTermVectorStored(),
- field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
- field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+ field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
+ field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
fp = new DocFieldProcessorPerField(this, fi);
fp.next = fieldHash[hashPos];
fieldHash[hashPos] = fp;
totalFieldCount++;
- if (totalFieldCount >= fieldHash.length/2)
+ if (totalFieldCount >= fieldHash.length/2) {
rehash();
+ }
} else {
fieldInfos.addOrUpdate(fp.fieldInfo.name, field.isIndexed(), field.isTermVectorStored(),
- field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
- field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+ field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
+ field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
}
if (thisFieldGen != fp.lastGen) {
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java Thu Jun 9 18:00:54 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -34,17 +33,17 @@ final class DocFieldProcessorPerField {
int lastGen = -1;
int fieldCount;
- Fieldable[] fields = new Fieldable[1];
+ IndexableField[] fields = new IndexableField[1];
public DocFieldProcessorPerField(final DocFieldProcessor docFieldProcessor, final FieldInfo fieldInfo) {
this.consumer = docFieldProcessor.consumer.addField(fieldInfo);
this.fieldInfo = fieldInfo;
}
- public void addField(Fieldable field) {
+ public void addField(IndexableField field) {
if (fieldCount == fields.length) {
int newSize = ArrayUtil.oversize(fieldCount + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- Fieldable[] newArray = new Fieldable[newSize];
+ IndexableField[] newArray = new IndexableField[newSize];
System.arraycopy(fields, 0, newArray, 0, fieldCount);
fields = newArray;
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java Thu Jun 9 18:00:54 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -61,16 +60,16 @@ final class DocInverterPerField extends
}
@Override
- public void processFields(final Fieldable[] fields,
+ public void processFields(final IndexableField[] fields,
final int count) throws IOException {
- fieldState.reset(docState.doc.getBoost());
+ fieldState.reset();
final boolean doInvert = consumer.start(fields, count);
for(int i=0;i<count;i++) {
- final Fieldable field = fields[i];
+ final IndexableField field = fields[i];
// TODO FI: this should be "genericized" to querying
// consumer if it wants to see this particular field
@@ -80,6 +79,9 @@ final class DocInverterPerField extends
if (i > 0)
fieldState.position += docState.analyzer.getPositionIncrementGap(fieldInfo.name);
+ // nocommit -- this logic should be outside of
+ // indexer
+
if (!field.isTokenized()) { // un-tokenized field
String stringValue = field.stringValue();
final int valueLength = stringValue.length();
@@ -103,17 +105,17 @@ final class DocInverterPerField extends
final TokenStream stream;
final TokenStream streamValue = field.tokenStreamValue();
- if (streamValue != null)
+ if (streamValue != null) {
stream = streamValue;
- else {
+ } else {
// the field does not have a TokenStream,
// so we have to obtain one from the analyzer
final Reader reader; // find or make Reader
final Reader readerValue = field.readerValue();
- if (readerValue != null)
+ if (readerValue != null) {
reader = readerValue;
- else {
+ } else {
String stringValue = field.stringValue();
if (stringValue == null) {
throw new IllegalArgumentException("field must have either TokenStream, String or Reader value");
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Thu Jun 9 18:00:54 2011
@@ -27,7 +27,6 @@ import java.util.Queue;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
@@ -321,7 +320,7 @@ final class DocumentsWriter {
return maybeMerge;
}
- boolean updateDocuments(final Iterable<Document> docs, final Analyzer analyzer,
+ boolean updateDocuments(final Iterable<? extends Iterable<? extends IndexableField>> docs, final Analyzer analyzer,
final Term delTerm) throws CorruptIndexException, IOException {
boolean maybeMerge = preUpdate();
@@ -352,7 +351,7 @@ final class DocumentsWriter {
return postUpdate(flushingDWPT, maybeMerge);
}
- boolean updateDocument(final Document doc, final Analyzer analyzer,
+ boolean updateDocument(final Iterable<? extends IndexableField> doc, final Analyzer analyzer,
final Term delTerm) throws CorruptIndexException, IOException {
boolean maybeMerge = preUpdate();
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Thu Jun 9 18:00:54 2011
@@ -26,7 +26,6 @@ import java.text.NumberFormat;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.store.Directory;
@@ -87,7 +86,7 @@ public class DocumentsWriterPerThread {
PrintStream infoStream;
SimilarityProvider similarityProvider;
int docID;
- Document doc;
+ Iterable<? extends IndexableField> doc;
String maxTermPrefix;
DocState(DocumentsWriterPerThread docWriter) {
@@ -209,7 +208,7 @@ public class DocumentsWriterPerThread {
return retval;
}
- public void updateDocument(Document doc, Analyzer analyzer, Term delTerm) throws IOException {
+ public void updateDocument(Iterable<? extends IndexableField> doc, Analyzer analyzer, Term delTerm) throws IOException {
assert writer.testPoint("DocumentsWriterPerThread addDocument start");
assert deleteQueue != null;
docState.doc = doc;
@@ -253,7 +252,7 @@ public class DocumentsWriterPerThread {
finishDocument(delTerm);
}
- public int updateDocuments(Iterable<Document> docs, Analyzer analyzer, Term delTerm) throws IOException {
+ public int updateDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer, Term delTerm) throws IOException {
assert writer.testPoint("DocumentsWriterPerThread addDocuments start");
assert deleteQueue != null;
docState.analyzer = analyzer;
@@ -265,7 +264,7 @@ public class DocumentsWriterPerThread {
int docCount = 0;
try {
- for(Document doc : docs) {
+ for(Iterable<? extends IndexableField> doc : docs) {
docState.doc = doc;
docState.docID = numDocsInRAM;
docCount++;
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInvertState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInvertState.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInvertState.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldInvertState.java Thu Jun 9 18:00:54 2011
@@ -49,13 +49,13 @@ public final class FieldInvertState {
* Re-initialize the state, using this boost value.
* @param docBoost boost value to use.
*/
- void reset(float docBoost) {
+ void reset() {
position = 0;
length = 0;
numOverlap = 0;
offset = 0;
maxTermFrequency = 0;
- boost = docBoost;
+ boost = 1.0f;
attributeSource = null;
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsReader.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsReader.java Thu Jun 9 18:00:54 2011
@@ -17,23 +17,23 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Reader;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.AbstractField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.NumericField;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CloseableThreadLocal;
-import java.io.IOException;
-import java.io.Reader;
-
/**
* Class responsible for access to stored document fields.
* <p/>
@@ -407,10 +407,10 @@ public final class FieldsReader implemen
}
/**
- * A Lazy implementation of Fieldable that defers loading of fields until asked for, instead of when the Document is
+ * A Lazy field implementation that defers loading of fields until asked for, instead of when the Document is
* loaded.
*/
- private class LazyField extends AbstractField implements Fieldable {
+ private class LazyField extends AbstractField {
private int toRead;
private long pointer;
private final boolean cacheResult;
@@ -437,6 +437,14 @@ public final class FieldsReader implemen
lazy = true;
}
+ public Number getNumericValue() {
+ return null;
+ }
+
+ public NumericField.DataType getDataType() {
+ return null;
+ }
+
private IndexInput getFieldStream() {
IndexInput localFieldsStream = fieldsStreamTL.get();
if (localFieldsStream == null) {
@@ -491,8 +499,7 @@ public final class FieldsReader implemen
}
}
- @Override
- public byte[] getBinaryValue(byte[] result) {
+ private byte[] getBinaryValue(byte[] result) {
ensureOpen();
if (isBinary) {
@@ -527,5 +534,15 @@ public final class FieldsReader implemen
} else
return null;
}
+
+ @Override
+ public BytesRef binaryValue(BytesRef reuse) {
+ final byte[] bytes = getBinaryValue(reuse != null ? reuse.bytes : null);
+ if (bytes != null) {
+ return new BytesRef(bytes, 0, bytes.length);
+ } else {
+ return null;
+ }
+ }
}
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsWriter.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsWriter.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FieldsWriter.java Thu Jun 9 18:00:54 2011
@@ -17,14 +17,11 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.List;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.NumericField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
final class FieldsWriter {
@@ -137,15 +134,22 @@ final class FieldsWriter {
}
}
- final void writeField(int fieldNumber, Fieldable field) throws IOException {
+ final void writeField(int fieldNumber, IndexableField field) throws IOException {
fieldsStream.writeVInt(fieldNumber);
int bits = 0;
- if (field.isTokenized())
+ // nocommit -- when we decouple analysis we should stop
+ // recording this:
+ if (field.isIndexed() && field.isTokenized()) {
bits |= FIELD_IS_TOKENIZED;
- if (field.isBinary())
- bits |= FIELD_IS_BINARY;
- if (field instanceof NumericField) {
- switch (((NumericField) field).getDataType()) {
+ }
+ final BytesRef bytes;
+ final String string;
+ // nocommit -- maybe a field should serialize itself?
+ // this way we don't bake into indexer all these
+ // specific encodings for different fields? and apps
+ // can customize...
+ if (field.isNumeric()) {
+ switch (field.getDataType()) {
case INT:
bits |= FIELD_IS_NUMERIC_INT; break;
case LONG:
@@ -157,23 +161,31 @@ final class FieldsWriter {
default:
assert false : "Should never get here";
}
+ string = null;
+ bytes = null;
+ } else {
+ bytes = field.binaryValue(null);
+ if (bytes != null) {
+ bits |= FIELD_IS_BINARY;
+ string = null;
+ } else {
+ string = field.stringValue();
+ }
}
+
fieldsStream.writeByte((byte) bits);
- if (field.isBinary()) {
- final byte[] data;
- final int len;
- final int offset;
- data = field.getBinaryValue();
- len = field.getBinaryLength();
- offset = field.getBinaryOffset();
-
- fieldsStream.writeVInt(len);
- fieldsStream.writeBytes(data, offset, len);
- } else if (field instanceof NumericField) {
- final NumericField nf = (NumericField) field;
- final Number n = nf.getNumericValue();
- switch (nf.getDataType()) {
+ if (bytes != null) {
+ fieldsStream.writeVInt(bytes.length);
+ fieldsStream.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ } else if (string != null) {
+ fieldsStream.writeString(field.stringValue());
+ } else {
+ final Number n = field.getNumericValue();
+ if (n == null) {
+ throw new IllegalArgumentException("field " + field.name() + " is stored but does not have binaryValue, stringValue nor numericValue");
+ }
+ switch (field.getDataType()) {
case INT:
fieldsStream.writeInt(n.intValue()); break;
case LONG:
@@ -185,8 +197,6 @@ final class FieldsWriter {
default:
assert false : "Should never get here";
}
- } else {
- fieldsStream.writeString(field.stringValue());
}
}
@@ -206,21 +216,21 @@ final class FieldsWriter {
assert fieldsStream.getFilePointer() == position;
}
- final void addDocument(Document doc, FieldInfos fieldInfos) throws IOException {
+ final void addDocument(Iterable<? extends IndexableField> doc, FieldInfos fieldInfos) throws IOException {
indexStream.writeLong(fieldsStream.getFilePointer());
int storedCount = 0;
- List<Fieldable> fields = doc.getFields();
- for (Fieldable field : fields) {
- if (field.isStored())
- storedCount++;
+ for (IndexableField field : doc) {
+ if (field.isStored()) {
+ storedCount++;
+ }
}
fieldsStream.writeVInt(storedCount);
-
- for (Fieldable field : fields) {
- if (field.isStored())
+ for (IndexableField field : doc) {
+ if (field.isStored()) {
writeField(fieldInfos.fieldNumber(field.name()), field);
+ }
}
}
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Thu Jun 9 18:00:54 2011
@@ -22,7 +22,6 @@ import java.util.Comparator;
import java.util.Map;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.PostingsConsumer;
import org.apache.lucene.index.codecs.TermStats;
@@ -81,15 +80,17 @@ final class FreqProxTermsWriterPerField
}
@Override
- boolean start(Fieldable[] fields, int count) {
- for(int i=0;i<count;i++)
- if (fields[i].isIndexed())
+ boolean start(IndexableField[] fields, int count) {
+ for(int i=0;i<count;i++) {
+ if (fields[i].isIndexed()) {
return true;
+ }
+ }
return false;
}
@Override
- void start(Fieldable f) {
+ void start(IndexableField f) {
if (fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
} else {
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexWriter.java Thu Jun 9 18:00:54 2011
@@ -36,7 +36,6 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -70,10 +69,10 @@ import org.apache.lucene.util.MapBackedS
new index if there is not already an index at the provided path
and otherwise open the existing index.</p>
- <p>In either case, documents are added with {@link #addDocument(Document)
+ <p>In either case, documents are added with {@link #addDocument(Iterable)
addDocument} and removed with {@link #deleteDocuments(Term)} or {@link
#deleteDocuments(Query)}. A document can be updated with {@link
- #updateDocument(Term, Document) updateDocument} (which just deletes
+ #updateDocument(Term, Iterable) updateDocument} (which just deletes
and then adds the entire document). When finished adding, deleting
and updating documents, {@link #close() close} should be called.</p>
@@ -1210,7 +1209,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void addDocument(Document doc) throws CorruptIndexException, IOException {
+ public void addDocument(Iterable<? extends IndexableField> doc) throws CorruptIndexException, IOException {
addDocument(doc, analyzer);
}
@@ -1218,7 +1217,7 @@ public class IndexWriter implements Clos
* Adds a document to this index, using the provided analyzer instead of the
* value of {@link #getAnalyzer()}.
*
- * <p>See {@link #addDocument(Document)} for details on
+ * <p>See {@link #addDocument(Iterable)} for details on
* index and IndexWriter state after an Exception, and
* flushing/merging temporary free space requirements.</p>
*
@@ -1229,7 +1228,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void addDocument(Iterable<? extends IndexableField> doc, Analyzer analyzer) throws CorruptIndexException, IOException {
updateDocument(null, doc, analyzer);
}
@@ -1247,7 +1246,7 @@ public class IndexWriter implements Clos
* compression), in which case you may need to fully
* re-index your documents at that time.
*
- * <p>See {@link #addDocument(Document)} for details on
+ * <p>See {@link #addDocument(Iterable)} for details on
* index and IndexWriter state after an Exception, and
* flushing/merging temporary free space requirements.</p>
*
@@ -1267,7 +1266,7 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public void addDocuments(Iterable<Document> docs) throws CorruptIndexException, IOException {
+ public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws CorruptIndexException, IOException {
addDocuments(docs, analyzer);
}
@@ -1282,7 +1281,7 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public void addDocuments(Iterable<Document> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
updateDocuments(null, docs, analyzer);
}
@@ -1299,7 +1298,7 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public void updateDocuments(Term delTerm, Iterable<Document> docs) throws CorruptIndexException, IOException {
+ public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws CorruptIndexException, IOException {
updateDocuments(delTerm, docs, analyzer);
}
@@ -1317,7 +1316,7 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public void updateDocuments(Term delTerm, Iterable<Document> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
ensureOpen();
try {
boolean success = false;
@@ -1440,7 +1439,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void updateDocument(Term term, Document doc) throws CorruptIndexException, IOException {
+ public void updateDocument(Term term, Iterable<? extends IndexableField> doc) throws CorruptIndexException, IOException {
ensureOpen();
updateDocument(term, doc, getAnalyzer());
}
@@ -1463,7 +1462,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void updateDocument(Term term, Document doc, Analyzer analyzer)
+ public void updateDocument(Term term, Iterable<? extends IndexableField> doc, Analyzer analyzer)
throws CorruptIndexException, IOException {
ensureOpen();
try {
@@ -2867,7 +2866,7 @@ public class IndexWriter implements Clos
DocumentsWriter getDocsWriter() {
boolean test = false;
assert test = true;
- return test?docWriter: null;
+ return test ? docWriter : null;
}
/** Expert: Return the number of documents currently
Added: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexableField.java?rev=1134021&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexableField.java (added)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/IndexableField.java Thu Jun 9 18:00:54 2011
@@ -0,0 +1,68 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.util.BytesRef;
+
+// nocommit jdocs
+
+// nocommit maybe take this further and push analysis into Document
+
+// nocommit what to do about multi-valued fields? really,
+// indexer should not know?
+
+/** @lucene.experimental */
+public interface IndexableField {
+ // nocommit: attrs?
+ // nocommit: doc values?
+ // nocommit: sorted?
+
+ public String name();
+
+ // nocommit -- make sure doc multplies its own boost in here:
+ public float getBoost();
+
+ public boolean isStored();
+
+ // nocommit -- isBinary?
+ public BytesRef binaryValue(BytesRef reuse);
+ public String stringValue();
+ public Reader readerValue();
+ public TokenStream tokenStreamValue();
+
+ // Numeric field:
+ public boolean isNumeric();
+ public NumericField.DataType getDataType();
+ public Number getNumericValue();
+
+ // If this returns non-null then we index this field:
+ public boolean isIndexed();
+ // nocommit maybe remove? only needed because stored
+ // fields records this!
+ public boolean isTokenized();
+ public boolean getOmitNorms();
+ public boolean getOmitTermFreqAndPositions();
+
+ public boolean isTermVectorStored();
+ public boolean isStoreOffsetWithTermVector();
+ public boolean isStorePositionWithTermVector();
+}
\ No newline at end of file
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java Thu Jun 9 18:00:54 2011
@@ -19,24 +19,22 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
-
abstract class InvertedDocConsumerPerField {
- // Called once per field, and is given all Fieldable
+ // Called once per field, and is given all IndexableField
// occurrences for this field in the document. Return
// true if you wish to see inverted tokens for these
// fields:
- abstract boolean start(Fieldable[] fields, int count) throws IOException;
+ abstract boolean start(IndexableField[] fields, int count) throws IOException;
// Called before a field instance is being processed
- abstract void start(Fieldable field);
+ abstract void start(IndexableField field);
// Called once per inverted token
abstract void add() throws IOException;
- // Called once per field per document, after all Fieldable
- // occurrences are inverted
+ // Called once per field per document, after all IndexableFields
+ // are inverted
abstract void finish() throws IOException;
// Called on hitting an aborting exception
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java Thu Jun 9 18:00:54 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -40,12 +39,12 @@ final class StoredFieldsWriter {
}
private int numStoredFields;
- private Fieldable[] storedFields;
+ private IndexableField[] storedFields;
private int[] fieldNumbers;
public void reset() {
numStoredFields = 0;
- storedFields = new Fieldable[1];
+ storedFields = new IndexableField[1];
fieldNumbers = new int[1];
}
@@ -122,10 +121,10 @@ final class StoredFieldsWriter {
assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end");
}
- public void addField(Fieldable field, FieldInfo fieldInfo) throws IOException {
+ public void addField(IndexableField field, FieldInfo fieldInfo) throws IOException {
if (numStoredFields == storedFields.length) {
int newSize = ArrayUtil.oversize(numStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- Fieldable[] newArray = new Fieldable[newSize];
+ IndexableField[] newArray = new IndexableField[newSize];
System.arraycopy(storedFields, 0, newArray, 0, numStoredFields);
storedFields = newArray;
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java Thu Jun 9 18:00:54 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
@@ -55,13 +54,13 @@ final class TermVectorsTermsWriterPerFie
}
@Override
- boolean start(Fieldable[] fields, int count) {
+ boolean start(IndexableField[] fields, int count) {
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
for(int i=0;i<count;i++) {
- Fieldable field = fields[i];
+ IndexableField field = fields[i];
if (field.isIndexed() && field.isTermVectorStored()) {
doVectors = true;
doVectorPositions |= field.isStorePositionWithTermVector();
@@ -188,7 +187,7 @@ final class TermVectorsTermsWriterPerFie
}
@Override
- void start(Fieldable f) {
+ void start(IndexableField f) {
if (doVectorOffsets) {
offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
} else {
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java Thu Jun 9 18:00:54 2011
@@ -24,13 +24,11 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
-
abstract class TermsHashConsumerPerField {
- abstract boolean start(Fieldable[] fields, int count) throws IOException;
+ abstract boolean start(IndexableField[] fields, int count) throws IOException;
abstract void finish() throws IOException;
abstract void skippingLongTerm() throws IOException;
- abstract void start(Fieldable field);
+ abstract void start(IndexableField field);
abstract void newTerm(int termID) throws IOException;
abstract void addTerm(int termID) throws IOException;
abstract int getStreamCount();
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java Thu Jun 9 18:00:54 2011
@@ -22,7 +22,6 @@ import java.util.Comparator;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
@@ -116,7 +115,7 @@ final class TermsHashPerField extends In
private boolean doNextCall;
@Override
- void start(Fieldable f) {
+ void start(IndexableField f) {
termAtt = fieldState.attributeSource.getAttribute(TermToBytesRefAttribute.class);
termBytesRef = termAtt.getBytesRef();
consumer.start(f);
@@ -126,11 +125,12 @@ final class TermsHashPerField extends In
}
@Override
- boolean start(Fieldable[] fields, int count) throws IOException {
+ boolean start(IndexableField[] fields, int count) throws IOException {
doCall = consumer.start(fields, count);
bytesHash.reinit();
- if (nextPerField != null)
+ if (nextPerField != null) {
doNextCall = nextPerField.start(fields, count);
+ }
return doCall || doNextCall;
}
Modified: lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/search/Similarity.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/java/org/apache/lucene/search/Similarity.java Thu Jun 9 18:00:54 2011
@@ -555,7 +555,7 @@ public abstract class Similarity {
* and larger values when <code>state.getLength()</code> is small.
*
* <p>Note that the return values are computed under
- * {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document)}
+ * {@link org.apache.lucene.index.IndexWriter#addDocument(Iterable)}
* and then stored using
* {@link #encodeNormValue(float)}.
* Thus they have limited precision, and documents
Modified: lucene/dev/branches/fieldtype/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Thu Jun 9 18:00:54 2011
@@ -24,7 +24,6 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter; // javadoc
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -96,19 +95,19 @@ public class RandomIndexWriter implement
/**
* Adds a Document.
- * @see IndexWriter#addDocument(Document)
+ * @see IndexWriter#addDocument(IndexableDocument)
*/
- public void addDocument(final Document doc) throws IOException {
+ public <T extends IndexableField> void addDocument(final Iterable<T> doc) throws IOException {
if (r.nextInt(5) == 3) {
// TODO: maybe, we should simply buffer up added docs
// (but we need to clone them), and only when
// getReader, commit, etc. are called, we do an
// addDocuments? Would be better testing.
- w.addDocuments(new Iterable<Document>() {
+ w.addDocuments(new Iterable<Iterable<T>>() {
// @Override -- not until Java 1.6
- public Iterator<Document> iterator() {
- return new Iterator<Document>() {
+ public Iterator<Iterable<T>> iterator() {
+ return new Iterator<Iterable<T>>() {
boolean done;
// @Override -- not until Java 1.6
@@ -122,7 +121,7 @@ public class RandomIndexWriter implement
}
// @Override -- not until Java 1.6
- public Document next() {
+ public Iterable<T> next() {
if (done) {
throw new IllegalStateException();
}
@@ -152,27 +151,27 @@ public class RandomIndexWriter implement
}
}
- public void addDocuments(Iterable<Document> docs) throws IOException {
+ public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
w.addDocuments(docs);
maybeCommit();
}
- public void updateDocuments(Term delTerm, Iterable<Document> docs) throws IOException {
+ public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
w.updateDocuments(delTerm, docs);
maybeCommit();
}
/**
* Updates a document.
- * @see IndexWriter#updateDocument(Term, Document)
+ * @see IndexWriter#updateDocument(Term, IndexableDocument)
*/
- public void updateDocument(Term t, final Document doc) throws IOException {
+ public <T extends IndexableField> void updateDocument(Term t, final Iterable<T> doc) throws IOException {
if (r.nextInt(5) == 3) {
- w.updateDocuments(t, new Iterable<Document>() {
+ w.updateDocuments(t, new Iterable<Iterable<T>>() {
// @Override -- not until Java 1.6
- public Iterator<Document> iterator() {
- return new Iterator<Document>() {
+ public Iterator<Iterable<T>> iterator() {
+ return new Iterator<Iterable<T>>() {
boolean done;
// @Override -- not until Java 1.6
@@ -186,7 +185,7 @@ public class RandomIndexWriter implement
}
// @Override -- not until Java 1.6
- public Document next() {
+ public Iterable<T> next() {
if (done) {
throw new IllegalStateException();
}
Modified: lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Thu Jun 9 18:00:54 2011
@@ -148,10 +148,10 @@ public class TestFieldsReader extends Lu
assertTrue("field is null and it shouldn't be", field != null);
assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
- byte [] bytes = field.getBinaryValue();
+ byte [] bytes = field.binaryValue(null).bytes;
assertTrue("bytes is null and it shouldn't be", bytes != null);
assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
- assertTrue("calling binaryValue() twice should give same reference", field.getBinaryValue() == field.getBinaryValue());
+ assertTrue("calling binaryValue() twice should give same reference", field.binaryValue(null).bytes == field.binaryValue(null).bytes);
for (int i = 0; i < bytes.length; i++) {
assertTrue("byte[" + i + "] is mismatched", bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
@@ -217,9 +217,9 @@ public class TestFieldsReader extends Lu
field = doc.getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
assertTrue("field is null and it shouldn't be", field != null);
assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
- assertTrue("calling binaryValue() twice should give different references", field.getBinaryValue() != field.getBinaryValue());
+ assertTrue("calling binaryValue() twice should give different references", field.binaryValue(null).bytes != field.binaryValue(null).bytes);
- byte [] bytes = field.getBinaryValue();
+ byte [] bytes = field.binaryValue(null).bytes;
assertTrue("bytes is null and it shouldn't be", bytes != null);
assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
for (int i = 0; i < bytes.length; i++) {
@@ -375,9 +375,9 @@ public class TestFieldsReader extends Lu
assertTrue(f1.isBinary());
assertTrue(!f3.isBinary());
assertTrue(fb.isBinary());
- assertSizeEquals(2*DocHelper.FIELD_1_TEXT.length(), f1.getBinaryValue());
+ assertSizeEquals(2*DocHelper.FIELD_1_TEXT.length(), f1.binaryValue(null).bytes);
assertEquals(DocHelper.FIELD_3_TEXT, f3.stringValue());
- assertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.length, fb.getBinaryValue());
+ assertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.length, fb.binaryValue(null).bytes);
reader.close();
}
Modified: lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Thu Jun 9 18:00:54 2011
@@ -349,10 +349,10 @@ public class TestIndexReader extends Luc
assertEquals(1, fields.length);
Field b1 = fields[0];
assertTrue(b1.isBinary());
- byte[] data1 = b1.getBinaryValue();
- assertEquals(bin.length, b1.getBinaryLength());
+ BytesRef bytesRef = b1.binaryValue(null);
+ assertEquals(bin.length, bytesRef.length);
for (int i = 0; i < bin.length; i++) {
- assertEquals(bin[i], data1[i + b1.getBinaryOffset()]);
+ assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
}
Set<String> lazyFields = new HashSet<String>();
lazyFields.add("bin1");
@@ -363,11 +363,11 @@ public class TestIndexReader extends Luc
assertEquals(1, fieldables.length);
Fieldable fb1 = fieldables[0];
assertTrue(fb1.isBinary());
- assertEquals(bin.length, fb1.getBinaryLength());
- data1 = fb1.getBinaryValue();
- assertEquals(bin.length, fb1.getBinaryLength());
+ bytesRef = fb1.binaryValue(null);
+ assertEquals(bin.length, bytesRef.bytes.length);
+ assertEquals(bin.length, bytesRef.length);
for (int i = 0; i < bin.length; i++) {
- assertEquals(bin[i], data1[i + fb1.getBinaryOffset()]);
+ assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
}
reader.close();
// force optimize
@@ -383,10 +383,10 @@ public class TestIndexReader extends Luc
assertEquals(1, fields.length);
b1 = fields[0];
assertTrue(b1.isBinary());
- data1 = b1.getBinaryValue();
- assertEquals(bin.length, b1.getBinaryLength());
+ bytesRef = b1.binaryValue(null);
+ assertEquals(bin.length, bytesRef.length);
for (int i = 0; i < bin.length; i++) {
- assertEquals(bin[i], data1[i + b1.getBinaryOffset()]);
+ assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
}
reader.close();
dir.close();
Modified: lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Thu Jun 9 18:00:54 2011
@@ -972,11 +972,11 @@ public class TestIndexWriter extends Luc
Document doc = new Document();
Field f = new Field("binary", b, 10, 17);
- byte[] bx = f.getBinaryValue();
+ byte[] bx = f.binaryValue(null).bytes;
assertTrue(bx != null);
assertEquals(50, bx.length);
- assertEquals(10, f.getBinaryOffset());
- assertEquals(17, f.getBinaryLength());
+ assertEquals(10, f.binaryValue(null).offset);
+ assertEquals(17, f.binaryValue(null).length);
doc.add(f);
w.addDocument(doc);
w.close();
@@ -984,7 +984,7 @@ public class TestIndexWriter extends Luc
IndexReader ir = IndexReader.open(dir, true);
doc = ir.document(0);
f = doc.getField("binary");
- b = f.getBinaryValue();
+ b = f.binaryValue(null).bytes;
assertTrue(b != null);
assertEquals(17, b.length, 17);
assertEquals(87, b[0]);
@@ -1239,7 +1239,7 @@ public class TestIndexWriter extends Luc
IndexReader ir = IndexReader.open(dir, true);
doc = ir.document(0);
f = doc.getField("binary");
- b = f.getBinaryValue();
+ b = f.binaryValue(null).bytes;
assertTrue(b != null);
assertEquals(17, b.length, 17);
assertEquals(87, b[0]);
Modified: lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestDocBoost.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestDocBoost.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestDocBoost.java (original)
+++ lucene/dev/branches/fieldtype/lucene/src/test/org/apache/lucene/search/TestDocBoost.java Thu Jun 9 18:00:54 2011
@@ -28,12 +28,17 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Ignore;
+
/** Document boost unit test.
*
*
*/
public class TestDocBoost extends LuceneTestCase {
+ // nocommit -- turn back on once Iterable<IndexableField>
+ // mutiplies doc boost in again
+ @Ignore
public void testDocBoost() throws Exception {
Directory store = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, store, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
Modified: lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java (original)
+++ lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java Thu Jun 9 18:00:54 2011
@@ -17,9 +17,9 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.IndexableField;
import java.io.Reader;
import java.io.IOException;
@@ -60,7 +60,7 @@ public final class LimitTokenCountAnalyz
}
@Override
- public int getOffsetGap(Fieldable field) {
+ public int getOffsetGap(IndexableField field) {
return delegate.getOffsetGap(field);
}
Modified: lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/branches/fieldtype/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java Thu Jun 9 18:00:54 2011
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis.misce
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.IndexableField;
import java.io.Reader;
import java.io.IOException;
@@ -119,10 +119,11 @@ public final class PerFieldAnalyzerWrapp
/** Return the offsetGap from the analyzer assigned to field */
@Override
- public int getOffsetGap(Fieldable field) {
+ public int getOffsetGap(IndexableField field) {
Analyzer analyzer = analyzerMap.get(field.name());
- if (analyzer == null)
+ if (analyzer == null) {
analyzer = defaultAnalyzer;
+ }
return analyzer.getOffsetGap(field);
}
Modified: lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Thu Jun 9 18:00:54 2011
@@ -253,9 +253,9 @@ public class LukeRequestHandler extends
// TODO: this really should be "stored"
f.add( "internal", fieldable.stringValue() ); // may be a binary number
- byte[] arr = fieldable.getBinaryValue();
- if (arr != null) {
- f.add( "binary", Base64.byteArrayToBase64(arr, 0, arr.length));
+ BytesRef bytes = fieldable.binaryValue(null);
+ if (bytes != null) {
+ f.add( "binary", Base64.byteArrayToBase64(bytes.bytes, bytes.offset, bytes.length));
}
f.add( "boost", fieldable.getBoost() );
f.add( "docFreq", t.text()==null ? 0 : reader.docFreq( t ) ); // this can be 0 for non-indexed fields
Modified: lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java (original)
+++ lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/response/BinaryResponseWriter.java Thu Jun 9 18:00:54 2011
@@ -165,7 +165,7 @@ public class BinaryResponseWriter implem
if(sf != null) ft =sf.getType();
Object val;
if (ft == null) { // handle fields not in the schema
- if (f.isBinary()) val = f.getBinaryValue();
+ if (f.isBinary()) val = f.binaryValue(null).bytes;
else val = f.stringValue();
} else {
try {
Modified: lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/BinaryField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/BinaryField.java?rev=1134021&r1=1134020&r2=1134021&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/BinaryField.java (original)
+++ lucene/dev/branches/fieldtype/solr/src/java/org/apache/solr/schema/BinaryField.java Thu Jun 9 18:00:54 2011
@@ -17,15 +17,16 @@
package org.apache.solr.schema;
-import org.apache.solr.response.TextResponseWriter;
-import org.apache.solr.common.util.Base64;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.search.SortField;
-
import java.io.IOException;
import java.nio.ByteBuffer;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.response.TextResponseWriter;
+
public class BinaryField extends FieldType {
@@ -51,7 +52,8 @@ public class BinaryField extends FieldTy
@Override
public ByteBuffer toObject(Fieldable f) {
- return ByteBuffer.wrap(f.getBinaryValue(), f.getBinaryOffset(), f.getBinaryLength() ) ;
+ BytesRef bytes = f.binaryValue(null);
+ return ByteBuffer.wrap(bytes.bytes, bytes.offset, bytes.length);
}
@Override