You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/08/16 00:03:52 UTC
svn commit: r1158029 [5/15] - in /lucene/dev/branches/fieldtype_conflicted:
lucene/ lucene/contrib/ lucene/contrib/demo/src/java/org/apache/lucene/demo/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
lucene/contrib/highlighter/...
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FieldsReader.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FieldsReader.java Mon Aug 15 22:03:41 2011
@@ -17,19 +17,14 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.AbstractField;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.NumericField;
+import java.io.IOException;
+
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+<<<<<<<
import org.apache.lucene.util.CloseableThreadLocal;
import org.apache.lucene.util.IOUtils;
@@ -37,6 +32,8 @@ import java.io.Closeable;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
+=======
+>>>>>>>
/**
* Class responsible for access to stored document fields.
@@ -68,7 +65,6 @@ public final class FieldsReader implemen
// file. This will be 0 if we have our own private file.
private int docStoreOffset;
- private CloseableThreadLocal<IndexInput> fieldsStreamTL = new CloseableThreadLocal<IndexInput>();
private boolean isOriginal = false;
/** Returns a cloned FieldsReader that shares open
@@ -200,50 +196,52 @@ public final class FieldsReader implemen
indexStream.seek(FORMAT_SIZE + (docID + docStoreOffset) * 8L);
}
- public final Document doc(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+ public final void visitDocument(int n, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
seekIndex(n);
- long position = indexStream.readLong();
- fieldsStream.seek(position);
+ fieldsStream.seek(indexStream.readLong());
- Document doc = new Document();
- int numFields = fieldsStream.readVInt();
- out: for (int i = 0; i < numFields; i++) {
+ final int numFields = fieldsStream.readVInt();
+ for (int fieldIDX = 0; fieldIDX < numFields; fieldIDX++) {
int fieldNumber = fieldsStream.readVInt();
- FieldInfo fi = fieldInfos.fieldInfo(fieldNumber);
- FieldSelectorResult acceptField = fieldSelector == null ? FieldSelectorResult.LOAD : fieldSelector.accept(fi.name);
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
int bits = fieldsStream.readByte() & 0xFF;
- assert bits <= (FieldsWriter.FIELD_IS_NUMERIC_MASK | FieldsWriter.FIELD_IS_TOKENIZED | FieldsWriter.FIELD_IS_BINARY): "bits=" + Integer.toHexString(bits);
+ assert bits <= (FieldsWriter.FIELD_IS_NUMERIC_MASK | FieldsWriter.FIELD_IS_BINARY): "bits=" + Integer.toHexString(bits);
- boolean tokenize = (bits & FieldsWriter.FIELD_IS_TOKENIZED) != 0;
- boolean binary = (bits & FieldsWriter.FIELD_IS_BINARY) != 0;
+ final boolean binary = (bits & FieldsWriter.FIELD_IS_BINARY) != 0;
final int numeric = bits & FieldsWriter.FIELD_IS_NUMERIC_MASK;
- switch (acceptField) {
- case LOAD:
- addField(doc, fi, binary, tokenize, numeric);
+ final boolean doStop;
+ if (binary) {
+ final int numBytes = fieldsStream.readVInt();
+ doStop = visitor.binaryField(fieldInfo, fieldsStream, numBytes);
+ } else if (numeric != 0) {
+ switch(numeric) {
+ case FieldsWriter.FIELD_IS_NUMERIC_INT:
+ doStop = visitor.intField(fieldInfo, fieldsStream.readInt());
break;
- case LOAD_AND_BREAK:
- addField(doc, fi, binary, tokenize, numeric);
- break out; //Get out of this loop
- case LAZY_LOAD:
- addFieldLazy(doc, fi, binary, tokenize, true, numeric);
+ case FieldsWriter.FIELD_IS_NUMERIC_LONG:
+ doStop = visitor.longField(fieldInfo, fieldsStream.readLong());
break;
- case LATENT:
- addFieldLazy(doc, fi, binary, tokenize, false, numeric);
+ case FieldsWriter.FIELD_IS_NUMERIC_FLOAT:
+ doStop = visitor.floatField(fieldInfo, Float.intBitsToFloat(fieldsStream.readInt()));
break;
- case SIZE:
- skipFieldBytes(addFieldSize(doc, fi, binary, numeric));
+ case FieldsWriter.FIELD_IS_NUMERIC_DOUBLE:
+ doStop = visitor.doubleField(fieldInfo, Double.longBitsToDouble(fieldsStream.readLong()));
break;
- case SIZE_AND_BREAK:
- addFieldSize(doc, fi, binary, numeric);
- break out; //Get out of this loop
default:
- skipField(numeric);
+ throw new FieldReaderException("Invalid numeric type: " + Integer.toHexString(numeric));
+ }
+ } else {
+ // Text:
+ final int numUTF8Bytes = fieldsStream.readVInt();
+ doStop = visitor.stringField(fieldInfo, fieldsStream, numUTF8Bytes);
}
- }
- return doc;
+ if (doStop) {
+ return;
+ }
+ }
}
/** Returns the length in bytes of each raw document in a
@@ -271,6 +269,7 @@ public final class FieldsReader implemen
return fieldsStream;
}
+<<<<<<<
/**
* Skip the field. We still have to read some of the information about the field, but can skip past the actual content.
@@ -521,4 +520,6 @@ public final class FieldsReader implemen
return null;
}
}
+=======
+>>>>>>>
}
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FieldsWriter.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FieldsWriter.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FieldsWriter.java Mon Aug 15 22:03:41 2011
@@ -17,19 +17,16 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.List;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.NumericField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
final class FieldsWriter {
- static final int FIELD_IS_TOKENIZED = 1 << 0;
+ // NOTE: bit 0 is free here!
static final int FIELD_IS_BINARY = 1 << 1;
// the old bit 1 << 2 was compressed, is now left out
@@ -138,15 +135,17 @@ final class FieldsWriter {
}
}
- final void writeField(int fieldNumber, Fieldable field) throws IOException {
+ final void writeField(int fieldNumber, IndexableField field) throws IOException {
fieldsStream.writeVInt(fieldNumber);
int bits = 0;
- if (field.isTokenized())
- bits |= FIELD_IS_TOKENIZED;
- if (field.isBinary())
- bits |= FIELD_IS_BINARY;
- if (field instanceof NumericField) {
- switch (((NumericField) field).getDataType()) {
+ final BytesRef bytes;
+ final String string;
+ // nocommit -- maybe a field should serialize itself?
+ // this way we don't bake into indexer all these
+ // specific encodings for different fields? and apps
+ // can customize...
+ if (field.numeric()) {
+ switch (field.numericDataType()) {
case INT:
bits |= FIELD_IS_NUMERIC_INT; break;
case LONG:
@@ -158,23 +157,31 @@ final class FieldsWriter {
default:
assert false : "Should never get here";
}
+ string = null;
+ bytes = null;
+ } else {
+ bytes = field.binaryValue(null);
+ if (bytes != null) {
+ bits |= FIELD_IS_BINARY;
+ string = null;
+ } else {
+ string = field.stringValue();
+ }
}
+
fieldsStream.writeByte((byte) bits);
- if (field.isBinary()) {
- final byte[] data;
- final int len;
- final int offset;
- data = field.getBinaryValue();
- len = field.getBinaryLength();
- offset = field.getBinaryOffset();
-
- fieldsStream.writeVInt(len);
- fieldsStream.writeBytes(data, offset, len);
- } else if (field instanceof NumericField) {
- final NumericField nf = (NumericField) field;
- final Number n = nf.getNumericValue();
- switch (nf.getDataType()) {
+ if (bytes != null) {
+ fieldsStream.writeVInt(bytes.length);
+ fieldsStream.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ } else if (string != null) {
+ fieldsStream.writeString(field.stringValue());
+ } else {
+ final Number n = field.numericValue();
+ if (n == null) {
+ throw new IllegalArgumentException("field " + field.name() + " is stored but does not have binaryValue, stringValue nor numericValue");
+ }
+ switch (field.numericDataType()) {
case INT:
fieldsStream.writeInt(n.intValue()); break;
case LONG:
@@ -186,8 +193,6 @@ final class FieldsWriter {
default:
assert false : "Should never get here";
}
- } else {
- fieldsStream.writeString(field.stringValue());
}
}
@@ -207,21 +212,21 @@ final class FieldsWriter {
assert fieldsStream.getFilePointer() == position;
}
- final void addDocument(Document doc, FieldInfos fieldInfos) throws IOException {
+ final void addDocument(Iterable<? extends IndexableField> doc, FieldInfos fieldInfos) throws IOException {
indexStream.writeLong(fieldsStream.getFilePointer());
int storedCount = 0;
- List<Fieldable> fields = doc.getFields();
- for (Fieldable field : fields) {
- if (field.isStored())
- storedCount++;
+ for (IndexableField field : doc) {
+ if (field.stored()) {
+ storedCount++;
+ }
}
fieldsStream.writeVInt(storedCount);
-
- for (Fieldable field : fields) {
- if (field.isStored())
+ for (IndexableField field : doc) {
+ if (field.stored()) {
writeField(fieldInfos.fieldNumber(field.name()), field);
+ }
}
}
}
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Mon Aug 15 22:03:41 2011
@@ -17,9 +17,13 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+<<<<<<<
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.codecs.PerDocValues;
+=======
+import org.apache.lucene.index.IndexReader.ReaderContext;
+>>>>>>>
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -351,9 +355,9 @@ public class FilterIndexReader extends I
}
@Override
- public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+ public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
ensureOpen();
- return in.document(n, fieldSelector);
+ in.document(docID, visitor);
}
@Override
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Mon Aug 15 22:03:41 2011
@@ -22,8 +22,11 @@ import java.util.Comparator;
import java.util.Map;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+<<<<<<<
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+=======
+>>>>>>>
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.PostingsConsumer;
import org.apache.lucene.index.codecs.TermStats;
@@ -82,15 +85,17 @@ final class FreqProxTermsWriterPerField
}
@Override
- boolean start(Fieldable[] fields, int count) {
- for(int i=0;i<count;i++)
- if (fields[i].isIndexed())
+ boolean start(IndexableField[] fields, int count) {
+ for(int i=0;i<count;i++) {
+ if (fields[i].indexed()) {
return true;
+ }
+ }
return false;
}
@Override
- void start(Fieldable f) {
+ void start(IndexableField f) {
if (fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
} else {
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexReader.java Mon Aug 15 22:03:41 2011
@@ -17,29 +17,31 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.search.FieldCache; // javadocs
-import org.apache.lucene.search.Similarity;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
+<<<<<<<
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.values.IndexDocValues;
+=======
+import org.apache.lucene.search.FieldCache; // javadocs
+import org.apache.lucene.search.Similarity;
+>>>>>>>
import org.apache.lucene.store.*;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil; // for javadocs
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.Closeable;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-
/** IndexReader is an abstract class, providing an interface for accessing an
index. Search of an index is done entirely through this abstract interface,
so that any subclass which implements it is searchable.
@@ -859,7 +861,6 @@ public abstract class IndexReader implem
* @return array of term frequency vectors. May be null if no term vectors have been
* stored for the specified document.
* @throws IOException if index cannot be accessed
- * @see org.apache.lucene.document.Field.TermVector
*/
abstract public TermFreqVector[] getTermFreqVectors(int docNumber)
throws IOException;
@@ -877,7 +878,6 @@ public abstract class IndexReader implem
* @return term frequency vector May be null if field does not exist in the specified
* document or term vector was not stored.
* @throws IOException if index cannot be accessed
- * @see org.apache.lucene.document.Field.TermVector
*/
abstract public TermFreqVector getTermFreqVector(int docNumber, String field)
throws IOException;
@@ -946,9 +946,22 @@ public abstract class IndexReader implem
return maxDoc() - numDocs();
}
+ /** Expert: visits the fields of a stored document, for
+ * custom processing/loading of each field. If you
+ * simply want to load all fields, use {@link
+ * #document(int)}. If you want to load a subset, use
+ * {@link DocumentStoredFieldVisitor}. */
+ public abstract void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException;
+
+ // nocommit -- the new document(int docID) API should
+ // clearly advertise that only field types/values are
+ // preserved -- index time metadata like boost, omitNorm,
+ // IndexOptions, tokenized are not preserved
+
/**
* Returns the stored fields of the <code>n</code><sup>th</sup>
- * <code>Document</code> in this index.
+ * <code>Document</code> in this index. This is just
+ * sugar for using {@link DocumentStoredFieldVisitor}.
* <p>
* <b>NOTE:</b> for performance reasons, this method does not check if the
* requested document is deleted, and therefore asking for a deleted document
@@ -959,11 +972,14 @@ public abstract class IndexReader implem
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public Document document(int n) throws CorruptIndexException, IOException {
+ public org.apache.lucene.document.Document document(int docID) throws CorruptIndexException, IOException {
ensureOpen();
- return document(n, null);
+ final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
+ document(docID, visitor);
+ return visitor.getDocument();
}
+<<<<<<<
/**
* Get the {@link org.apache.lucene.document.Document} at the <code>n</code>
* <sup>th</sup> position. The {@link FieldSelector} may be used to determine
@@ -997,6 +1013,8 @@ public abstract class IndexReader implem
// TODO (1.5): When we convert to JDK 1.5 make this Set<String>
public abstract Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException;
+=======
+>>>>>>>
/** Returns true if any documents have been deleted */
public abstract boolean hasDeletions();
@@ -1017,8 +1035,13 @@ public abstract class IndexReader implem
public abstract byte[] norms(String field) throws IOException;
/** Expert: Resets the normalization factor for the named field of the named
+<<<<<<<
* document. By default, The norm represents the product of the field's {@link
* org.apache.lucene.document.Fieldable#setBoost(float) boost} and its
+=======
+ * document. The norm represents the product of the field's {@link
+ * org.apache.lucene.document.Field#setBoost(float) boost} and its
+>>>>>>>
* length normalization}. Thus, to preserve the length normalization
* values when resetting this, one should base the new value upon the old.
*
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon Aug 15 22:03:41 2011
@@ -36,7 +36,6 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -74,10 +73,10 @@ import org.apache.lucene.util.TwoPhaseCo
new index if there is not already an index at the provided path
and otherwise open the existing index.</p>
- <p>In either case, documents are added with {@link #addDocument(Document)
+ <p>In either case, documents are added with {@link #addDocument(Iterable)
addDocument} and removed with {@link #deleteDocuments(Term)} or {@link
#deleteDocuments(Query)}. A document can be updated with {@link
- #updateDocument(Term, Document) updateDocument} (which just deletes
+ #updateDocument(Term, Iterable) updateDocument} (which just deletes
and then adds the entire document). When finished adding, deleting
and updating documents, {@link #close() close} should be called.</p>
@@ -1283,7 +1282,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void addDocument(Document doc) throws CorruptIndexException, IOException {
+ public void addDocument(Iterable<? extends IndexableField> doc) throws CorruptIndexException, IOException {
addDocument(doc, analyzer);
}
@@ -1291,7 +1290,7 @@ public class IndexWriter implements Clos
* Adds a document to this index, using the provided analyzer instead of the
* value of {@link #getAnalyzer()}.
*
- * <p>See {@link #addDocument(Document)} for details on
+ * <p>See {@link #addDocument(Iterable)} for details on
* index and IndexWriter state after an Exception, and
* flushing/merging temporary free space requirements.</p>
*
@@ -1302,7 +1301,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void addDocument(Iterable<? extends IndexableField> doc, Analyzer analyzer) throws CorruptIndexException, IOException {
updateDocument(null, doc, analyzer);
}
@@ -1320,7 +1319,7 @@ public class IndexWriter implements Clos
* compression), in which case you may need to fully
* re-index your documents at that time.
*
- * <p>See {@link #addDocument(Document)} for details on
+ * <p>See {@link #addDocument(Iterable)} for details on
* index and IndexWriter state after an Exception, and
* flushing/merging temporary free space requirements.</p>
*
@@ -1340,7 +1339,7 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public void addDocuments(Iterable<Document> docs) throws CorruptIndexException, IOException {
+ public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws CorruptIndexException, IOException {
addDocuments(docs, analyzer);
}
@@ -1355,7 +1354,7 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public void addDocuments(Iterable<Document> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
updateDocuments(null, docs, analyzer);
}
@@ -1372,7 +1371,7 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public void updateDocuments(Term delTerm, Iterable<Document> docs) throws CorruptIndexException, IOException {
+ public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws CorruptIndexException, IOException {
updateDocuments(delTerm, docs, analyzer);
}
@@ -1390,7 +1389,7 @@ public class IndexWriter implements Clos
*
* @lucene.experimental
*/
- public void updateDocuments(Term delTerm, Iterable<Document> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
ensureOpen();
try {
boolean success = false;
@@ -1513,7 +1512,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void updateDocument(Term term, Document doc) throws CorruptIndexException, IOException {
+ public void updateDocument(Term term, Iterable<? extends IndexableField> doc) throws CorruptIndexException, IOException {
ensureOpen();
updateDocument(term, doc, getAnalyzer());
}
@@ -1536,7 +1535,7 @@ public class IndexWriter implements Clos
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void updateDocument(Term term, Document doc, Analyzer analyzer)
+ public void updateDocument(Term term, Iterable<? extends IndexableField> doc, Analyzer analyzer)
throws CorruptIndexException, IOException {
ensureOpen();
try {
@@ -3036,7 +3035,7 @@ public class IndexWriter implements Clos
DocumentsWriter getDocsWriter() {
boolean test = false;
assert test = true;
- return test?docWriter: null;
+ return test ? docWriter : null;
}
/** Expert: Return the number of documents currently
Added: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexableField.java?rev=1158029&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexableField.java (added)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/IndexableField.java Mon Aug 15 22:03:41 2011
@@ -0,0 +1,86 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.document.NumericField.DataType;
+import org.apache.lucene.util.BytesRef;
+
+// nocommit jdocs
+
+// nocommit maybe abstract class instead...? or maybe we
+// have versioned interfaces over time, so indexer can
+// consume JARs w/ older doc/field impls? IndexableField1,
+// IndexableField2, ...?
+
+// nocommit maybe take this further and push analysis into Document
+
+// nocommit what to do about multi-valued fields? really,
+// indexer should not know?
+
+// nocommit make test case showing how you can index
+// Iterable<IndexableField> that is not a Document
+// instance...
+
+/** @lucene.experimental */
+public interface IndexableField {
+ // nocommit: attrs?
+ // nocommit: doc values?
+ // nocommit: sorted?
+
+ public String name();
+
+ // NOTE: if doc/field impl has the notion of "doc level boost"
+ // it must be multiplied in w/ this field's boost
+ public float boost();
+
+ public boolean stored();
+
+ // nocommit -- isBinary?
+ public BytesRef binaryValue(BytesRef reuse);
+ public String stringValue();
+ public Reader readerValue();
+
+ // nocommit -- decouple analyzers here: field impl should
+ // go and ask analyzer for the token stream, so indexer
+ // doesn't have to ask for string/reader value and then consult
+ // analyzer
+ public TokenStream tokenStreamValue();
+
+ // Numeric field:
+ public boolean numeric();
+ public DataType numericDataType();
+ public Number numericValue();
+
+ // If this returns true then we index this field:
+ public boolean indexed();
+
+ // nocommit maybe remove? only needed because stored
+ // fields records this! (well, and because analysis isn't
+ // yet decoupled)
+ public boolean tokenized();
+ public boolean omitNorms();
+ public boolean omitTermFreqAndPositions();
+
+ public boolean storeTermVectors();
+ public boolean storeTermVectorOffsets();
+ public boolean storeTermVectorPositions();
+}
\ No newline at end of file
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java Mon Aug 15 22:03:41 2011
@@ -19,24 +19,22 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
-
abstract class InvertedDocConsumerPerField {
- // Called once per field, and is given all Fieldable
+ // Called once per field, and is given all IndexableField
// occurrences for this field in the document. Return
// true if you wish to see inverted tokens for these
// fields:
- abstract boolean start(Fieldable[] fields, int count) throws IOException;
+ abstract boolean start(IndexableField[] fields, int count) throws IOException;
// Called before a field instance is being processed
- abstract void start(Fieldable field);
+ abstract void start(IndexableField field);
// Called once per inverted token
abstract void add() throws IOException;
- // Called once per field per document, after all Fieldable
- // occurrences are inverted
+ // Called once per field per document, after all IndexableFields
+ // are inverted
abstract void finish() throws IOException;
// Called on hitting an aborting exception
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/MultiReader.java Mon Aug 15 22:03:41 2011
@@ -22,9 +22,12 @@ import java.util.Collection;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+<<<<<<<
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.codecs.PerDocValues;
+=======
+>>>>>>>
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
@@ -258,12 +261,11 @@ public class MultiReader extends IndexRe
return maxDoc;
}
- // inherit javadoc
@Override
- public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+ public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
ensureOpen();
- int i = readerIndex(n); // find segment num
- return subReaders[i].document(n - starts[i], fieldSelector); // dispatch to segment reader
+ int i = readerIndex(docID); // find segment num
+ subReaders[i].document(docID - starts[i], visitor); // dispatch to segment reader
}
@Override
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/ParallelReader.java Mon Aug 15 22:03:41 2011
@@ -17,12 +17,15 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+<<<<<<<
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.values.IndexDocValues;
+=======
+>>>>>>>
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.MapBackedSet;
@@ -351,30 +354,12 @@ public class ParallelReader extends Inde
hasDeletions = false;
}
- // append fields from storedFieldReaders
@Override
- public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+ public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
ensureOpen();
- Document result = new Document();
for (final IndexReader reader: storedFieldReaders) {
-
- boolean include = (fieldSelector==null);
- if (!include) {
- Collection<String> fields = readerToFields.get(reader);
- for (final String field : fields)
- if (fieldSelector.accept(field) != FieldSelectorResult.NO_LOAD) {
- include = true;
- break;
- }
- }
- if (include) {
- List<Fieldable> fields = reader.document(n, fieldSelector).getFields();
- for (Fieldable field : fields) {
- result.add(field);
- }
- }
+ reader.document(docID, visitor);
}
- return result;
}
// get all vectors
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/PersistentSnapshotDeletionPolicy.java Mon Aug 15 22:03:41 2011
@@ -25,9 +25,7 @@ import java.util.Map.Entry;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
@@ -71,12 +69,12 @@ public class PersistentSnapshotDeletionP
// index is allowed to have exactly one document or 0.
if (numDocs == 1) {
Document doc = r.document(r.maxDoc() - 1);
- Field sid = doc.getField(SNAPSHOTS_ID);
+ Field sid = (Field) doc.getField(SNAPSHOTS_ID);
if (sid == null) {
throw new IllegalStateException("directory is not a valid snapshots store!");
}
doc.removeField(SNAPSHOTS_ID);
- for (Fieldable f : doc.getFields()) {
+ for (IndexableField f : doc) {
snapshots.put(f.name(), f.stringValue());
}
} else if (numDocs != 0) {
@@ -189,12 +187,14 @@ public class PersistentSnapshotDeletionP
private void persistSnapshotInfos(String id, String segment) throws IOException {
writer.deleteAll();
Document d = new Document();
- d.add(new Field(SNAPSHOTS_ID, "", Store.YES, Index.NO));
+ FieldType ft = new FieldType();
+ ft.setStored(true);
+ d.add(new Field(SNAPSHOTS_ID, ft, ""));
for (Entry<String, String> e : super.getSnapshots().entrySet()) {
- d.add(new Field(e.getKey(), e.getValue(), Store.YES, Index.NO));
+ d.add(new Field(e.getKey(), ft, e.getValue()));
}
if (id != null) {
- d.add(new Field(id, segment, Store.YES, Index.NO));
+ d.add(new Field(id, ft, segment));
}
writer.addDocument(d);
writer.commit();
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Mon Aug 15 22:03:41 2011
@@ -335,6 +335,10 @@ final class SegmentMerger {
// skip deleted docs
continue;
}
+ // TODO: this could be more efficient using
+ // FieldVisitor instead of loading/writing entire
+ // doc; ie we just have to renumber the field number
+ // on the fly?
// NOTE: it's very important to first assign to doc then pass it to
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
Document doc = reader.reader.document(j);
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/SegmentReader.java Mon Aug 15 22:03:41 2011
@@ -27,8 +27,14 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+<<<<<<<
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
+=======
+import org.apache.lucene.store.BufferedIndexInput;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+>>>>>>>
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.Directory;
@@ -455,10 +461,9 @@ public class SegmentReader extends Index
return core.fieldInfos;
}
- @Override
- public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+ public void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException {
ensureOpen();
- return getFieldsReader().doc(n, fieldSelector);
+ getFieldsReader().visitDocument(docID, visitor);
}
@Override
Added: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java?rev=1158029&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java (added)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java Mon Aug 15 22:03:41 2011
@@ -0,0 +1,87 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.store.IndexInput;
+
+/**
+ * Expert: provides a low-level means of accessing the stored field
+ * values in an index. See {@link IndexReader#document(int,
+ * StoredFieldVisitor)}.
+ *
+ * See {@link DocumentStoredFieldVisitor}, which is a
+ * <code>StoredFieldVisitor</code> that builds the
+ * {@link Document} containing all stored fields. This is
+ * used by {@link IndexReader#document(int)}.
+ *
+ * @lucene.experimental */
+
+public class StoredFieldVisitor {
+ /** Process a binary field. Note that if you want to
+ * skip the field you must seek the IndexInput
+ * (e.g., call <code>in.seek(numUTF8Bytes + in.getFilePointer()</code>)
+ *
+ * <p>Return true to stop loading fields. */
+ public boolean binaryField(FieldInfo fieldInfo, IndexInput in, int numBytes) throws IOException {
+ in.seek(in.getFilePointer() + numBytes);
+ return false;
+ }
+
+ /** Process a string field by reading numUTF8Bytes.
+ * Note that if you want to skip the field you must
+ * seek the IndexInput as if you had read numBytes by
+ * (e.g., call <code>in.seek(numUTF8Bytes + in.getFilePointer()</code>)
+ *
+ * <p>Return true to stop loading fields. */
+ public boolean stringField(FieldInfo fieldInfo, IndexInput in, int numUTF8Bytes) throws IOException {
+ in.seek(in.getFilePointer() + numUTF8Bytes);
+ return false;
+ }
+
+ /** Process a int numeric field.
+ *
+ * <p>Return true to stop loading fields. */
+ public boolean intField(FieldInfo fieldInfo, int value) throws IOException {
+ return false;
+ }
+
+ /** Process a long numeric field.
+ *
+ * <p>Return true to stop loading fields. */
+ public boolean longField(FieldInfo fieldInfo, long value) throws IOException {
+ return false;
+ }
+
+ /** Process a float numeric field.
+ *
+ * <p>Return true to stop loading fields. */
+ public boolean floatField(FieldInfo fieldInfo, float value) throws IOException {
+ return false;
+ }
+
+ /** Process a double numeric field.
+ *
+ * <p>Return true to stop loading fields. */
+ public boolean doubleField(FieldInfo fieldInfo, double value) throws IOException {
+ return false;
+ }
+}
+
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java Mon Aug 15 22:03:41 2011
@@ -19,8 +19,11 @@ package org.apache.lucene.index;
import java.io.IOException;
+<<<<<<<
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.store.IOContext;
+=======
+>>>>>>>
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -41,12 +44,12 @@ final class StoredFieldsWriter {
}
private int numStoredFields;
- private Fieldable[] storedFields;
+ private IndexableField[] storedFields;
private int[] fieldNumbers;
public void reset() {
numStoredFields = 0;
- storedFields = new Fieldable[1];
+ storedFields = new IndexableField[1];
fieldNumbers = new int[1];
}
@@ -123,10 +126,10 @@ final class StoredFieldsWriter {
assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end");
}
- public void addField(Fieldable field, FieldInfo fieldInfo) throws IOException {
+ public void addField(IndexableField field, FieldInfo fieldInfo) throws IOException {
if (numStoredFields == storedFields.length) {
int newSize = ArrayUtil.oversize(numStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- Fieldable[] newArray = new Fieldable[newSize];
+ IndexableField[] newArray = new IndexableField[newSize];
System.arraycopy(storedFields, 0, newArray, 0, numStoredFields);
storedFields = newArray;
}
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermFreqVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermFreqVector.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermFreqVector.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermFreqVector.java Mon Aug 15 22:03:41 2011
@@ -26,7 +26,7 @@ import org.apache.lucene.util.BytesRef;
*/
public interface TermFreqVector {
/**
- * The {@link org.apache.lucene.document.Fieldable} name.
+ * The {@link org.apache.lucene.index.IndexableField} name.
* @return The name of the field this vector is associated with.
*
*/
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java Mon Aug 15 22:03:41 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
@@ -55,17 +54,17 @@ final class TermVectorsTermsWriterPerFie
}
@Override
- boolean start(Fieldable[] fields, int count) {
+ boolean start(IndexableField[] fields, int count) {
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
for(int i=0;i<count;i++) {
- Fieldable field = fields[i];
- if (field.isIndexed() && field.isTermVectorStored()) {
+ IndexableField field = fields[i];
+ if (field.indexed() && field.storeTermVectors()) {
doVectors = true;
- doVectorPositions |= field.isStorePositionWithTermVector();
- doVectorOffsets |= field.isStoreOffsetWithTermVector();
+ doVectorPositions |= field.storeTermVectorPositions();
+ doVectorOffsets |= field.storeTermVectorOffsets();
}
}
@@ -188,7 +187,7 @@ final class TermVectorsTermsWriterPerFie
}
@Override
- void start(Fieldable f) {
+ void start(IndexableField f) {
if (doVectorOffsets) {
offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
} else {
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java Mon Aug 15 22:03:41 2011
@@ -24,13 +24,11 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
-
abstract class TermsHashConsumerPerField {
- abstract boolean start(Fieldable[] fields, int count) throws IOException;
+ abstract boolean start(IndexableField[] fields, int count) throws IOException;
abstract void finish() throws IOException;
abstract void skippingLongTerm() throws IOException;
- abstract void start(Fieldable field);
+ abstract void start(IndexableField field);
abstract void newTerm(int termID) throws IOException;
abstract void addTerm(int termID) throws IOException;
abstract int getStreamCount();
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java Mon Aug 15 22:03:41 2011
@@ -22,7 +22,6 @@ import java.util.Comparator;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
@@ -116,7 +115,7 @@ final class TermsHashPerField extends In
private boolean doNextCall;
@Override
- void start(Fieldable f) {
+ void start(IndexableField f) {
termAtt = fieldState.attributeSource.getAttribute(TermToBytesRefAttribute.class);
termBytesRef = termAtt.getBytesRef();
consumer.start(f);
@@ -126,11 +125,12 @@ final class TermsHashPerField extends In
}
@Override
- boolean start(Fieldable[] fields, int count) throws IOException {
+ boolean start(IndexableField[] fields, int count) throws IOException {
doCall = consumer.start(fields, count);
bytesHash.reinit();
- if (nextPerField != null)
+ if (nextPerField != null) {
doNextCall = nextPerField.start(fields, count);
+ }
return doCall || doNextCall;
}
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/FieldCache.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/FieldCache.java Mon Aug 15 22:03:41 2011
@@ -24,7 +24,7 @@ import org.apache.lucene.search.cache.Ca
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.document.NumericField; // for javadocs
+import org.apache.lucene.document.NumericField;
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
import org.apache.lucene.util.packed.PackedInts;
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java Mon Aug 15 22:03:41 2011
@@ -23,7 +23,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.document.NumericField; // for javadocs
+import org.apache.lucene.document.NumericField;
/**
* A range filter built on top of a cached single term field (in {@link FieldCache}).
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Mon Aug 15 22:03:41 2011
@@ -31,11 +31,11 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Weight.ScorerContext;
import org.apache.lucene.store.Directory;
@@ -239,16 +239,16 @@ public class IndexSearcher implements Cl
}
}
- /* Sugar for .getIndexReader().document(docID) */
+ /* Sugar for <code>.getIndexReader().document(docID)</code> */
public Document doc(int docID) throws CorruptIndexException, IOException {
return reader.document(docID);
}
-
- /* Sugar for .getIndexReader().document(docID, fieldSelector) */
- public Document doc(int docID, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
- return reader.document(docID, fieldSelector);
+
+ /* Sugar for <code>.getIndexReader().document(docID, fieldVisitor)</code> */
+ public void doc(int docID, StoredFieldVisitor fieldVisitor) throws CorruptIndexException, IOException {
+ reader.document(docID, fieldVisitor);
}
-
+
/** Expert: Set the SimilarityProvider implementation used by this Searcher.
*
*/
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/NumericRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/NumericRangeFilter.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/NumericRangeFilter.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/NumericRangeFilter.java Mon Aug 15 22:03:41 2011
@@ -18,7 +18,7 @@ package org.apache.lucene.search;
*/
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
-import org.apache.lucene.document.NumericField; // for javadocs
+import org.apache.lucene.document.NumericField;
import org.apache.lucene.util.NumericUtils; // for javadocs
/**
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/NumericRangeQuery.java Mon Aug 15 22:03:41 2011
@@ -22,7 +22,7 @@ import java.util.LinkedList;
import java.util.Comparator;
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
-import org.apache.lucene.document.NumericField; // for javadocs
+import org.apache.lucene.document.NumericField;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.index.Terms;
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/Similarity.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/java/org/apache/lucene/search/Similarity.java Mon Aug 15 22:03:41 2011
@@ -222,3 +222,43 @@ public abstract class Similarity {
public abstract void normalize(float queryNorm, float topLevelBoost);
}
}
+<<<<<<<
+=======
+ * <b><i>norm(t,d)</i></b> encapsulates a few (indexing time) boost and length factors:
+ *
+ * <ul>
+ * <li><b>Field boost</b> - set by calling
+ * {@link org.apache.lucene.document.Field#setBoost(float) field.setBoost()}
+ * before adding the field to a document.
+ * </li>
+ * <li><b>lengthNorm</b> - computed
+>>>>>>>
+<<<<<<<
+=======
+ * <table cellpadding="1" cellspacing="0" border="0"n align="center">
+ * <tr>
+ * <td valign="middle" align="right" rowspan="1">
+ * lengthNorm
+ * ·
+ * </td>
+>>>>>>>
+<<<<<<<
+=======
+ * <big><big><big>∏</big></big></big>
+ * </td>
+ * <td valign="middle" align="right" rowspan="1">
+ * {@link org.apache.lucene.index.IndexableField#boost() f.getBoost}()
+ * </td>
+ * </tr>
+ * <tr valigh="top">
+>>>>>>>
+<<<<<<<
+=======
+ * and larger values when <code>state.getLength()</code> is small.
+ *
+ * <p>Note that the return values are computed under
+ * {@link org.apache.lucene.index.IndexWriter#addDocument(Iterable)}
+ * and then stored using
+ * {@link #encodeNormValue(float)}.
+ * Thus they have limited precision, and documents
+>>>>>>>
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java Mon Aug 15 22:03:41 2011
@@ -26,10 +26,16 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.document.BinaryField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+<<<<<<<
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+=======
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+>>>>>>>
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -37,63 +43,114 @@ import org.apache.lucene.util.LuceneTest
import static org.apache.lucene.util.LuceneTestCase.TEST_VERSION_CURRENT;
class DocHelper {
+
+ public static final FieldType customType;
public static final String FIELD_1_TEXT = "field one text";
public static final String TEXT_FIELD_1_KEY = "textField1";
- public static Field textField1 = new Field(TEXT_FIELD_1_KEY, FIELD_1_TEXT,
- Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO);
-
+ public static Field textField1;
+ static {
+ customType = new FieldType(TextField.TYPE_UNSTORED);
+ customType.setStored(true);
+ textField1 = new Field(TEXT_FIELD_1_KEY, customType, FIELD_1_TEXT);
+ }
+
+ public static final FieldType customType2;
public static final String FIELD_2_TEXT = "field field field two text";
//Fields will be lexicographically sorted. So, the order is: field, text, two
public static final int [] FIELD_2_FREQS = {3, 1, 1};
public static final String TEXT_FIELD_2_KEY = "textField2";
- public static Field textField2 = new Field(TEXT_FIELD_2_KEY, FIELD_2_TEXT, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
+ public static Field textField2;
+ static {
+ customType2 = new FieldType(TextField.TYPE_UNSTORED);
+ customType2.setStored(true);
+ customType2.setStoreTermVectors(true);
+ customType2.setStoreTermVectorPositions(true);
+ customType2.setStoreTermVectorOffsets(true);
+ textField2 = new Field(TEXT_FIELD_2_KEY, customType2, FIELD_2_TEXT);
+ }
+ public static final FieldType customType3;
public static final String FIELD_3_TEXT = "aaaNoNorms aaaNoNorms bbbNoNorms";
public static final String TEXT_FIELD_3_KEY = "textField3";
- public static Field textField3 = new Field(TEXT_FIELD_3_KEY, FIELD_3_TEXT, Field.Store.YES, Field.Index.ANALYZED);
- static { textField3.setOmitNorms(true); }
+ public static Field textField3;
+
+ static {
+ customType3 = new FieldType(TextField.TYPE_UNSTORED);
+ customType3.setStored(true);
+ customType3.setOmitNorms(true);
+ textField3 = new Field(TEXT_FIELD_3_KEY, customType3, FIELD_3_TEXT);
+ }
+ public static final FieldType customType4;
public static final String KEYWORD_TEXT = "Keyword";
public static final String KEYWORD_FIELD_KEY = "keyField";
- public static Field keyField = new Field(KEYWORD_FIELD_KEY, KEYWORD_TEXT,
- Field.Store.YES, Field.Index.NOT_ANALYZED);
+ public static Field keyField;
+ static {
+ customType4 = new FieldType(TextField.TYPE_UNSTORED);
+ customType4.setStored(true);
+ customType4.setTokenized(false);
+ keyField = new Field(KEYWORD_FIELD_KEY, customType4, KEYWORD_TEXT);
+ }
+ public static final FieldType customType5;
public static final String NO_NORMS_TEXT = "omitNormsText";
public static final String NO_NORMS_KEY = "omitNorms";
- public static Field noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT,
- Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
+ public static Field noNormsField;
+ static {
+ customType5 = new FieldType(TextField.TYPE_UNSTORED);
+ customType5.setOmitNorms(true);
+ customType5.setStored(true);
+ customType5.setTokenized(false);
+ noNormsField = new Field(NO_NORMS_KEY, customType5, NO_NORMS_TEXT);
+ }
+ public static final FieldType customType6;
public static final String NO_TF_TEXT = "analyzed with no tf and positions";
public static final String NO_TF_KEY = "omitTermFreqAndPositions";
- public static Field noTFField = new Field(NO_TF_KEY, NO_TF_TEXT,
- Field.Store.YES, Field.Index.ANALYZED);
+ public static Field noTFField;
static {
+<<<<<<<
noTFField.setIndexOptions(IndexOptions.DOCS_ONLY);
+=======
+ customType6 = new FieldType(TextField.TYPE_UNSTORED);
+ customType6.setOmitTermFreqAndPositions(true);
+ customType6.setStored(true);
+ noTFField = new Field(NO_TF_KEY, customType6, NO_TF_TEXT);
+>>>>>>>
}
+ public static final FieldType customType7;
public static final String UNINDEXED_FIELD_TEXT = "unindexed field text";
public static final String UNINDEXED_FIELD_KEY = "unIndField";
- public static Field unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT,
- Field.Store.YES, Field.Index.NO);
+ public static Field unIndField;
+ static {
+ customType7 = new FieldType();
+ customType7.setStored(true);
+ unIndField = new Field(UNINDEXED_FIELD_KEY, customType7, UNINDEXED_FIELD_TEXT);
+ }
public static final String UNSTORED_1_FIELD_TEXT = "unstored field text";
public static final String UNSTORED_FIELD_1_KEY = "unStoredField1";
- public static Field unStoredField1 = new Field(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT,
- Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.NO);
+ public static Field unStoredField1 = new Field(UNSTORED_FIELD_1_KEY, TextField.TYPE_UNSTORED, UNSTORED_1_FIELD_TEXT);
+ public static final FieldType customType8;
public static final String UNSTORED_2_FIELD_TEXT = "unstored field text";
public static final String UNSTORED_FIELD_2_KEY = "unStoredField2";
- public static Field unStoredField2 = new Field(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT,
- Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.YES);
+ public static Field unStoredField2;
+ static {
+ customType8 = new FieldType(TextField.TYPE_UNSTORED);
+ customType8.setStoreTermVectors(true);
+ unStoredField2 = new Field(UNSTORED_FIELD_2_KEY, customType8, UNSTORED_2_FIELD_TEXT);
+ }
public static final String LAZY_FIELD_BINARY_KEY = "lazyFieldBinary";
public static byte [] LAZY_FIELD_BINARY_BYTES;
public static Field lazyFieldBinary;
-
+
public static final String LAZY_FIELD_KEY = "lazyField";
public static final String LAZY_FIELD_TEXT = "These are some field bytes";
- public static Field lazyField = new Field(LAZY_FIELD_KEY, LAZY_FIELD_TEXT, Field.Store.YES, Field.Index.ANALYZED);
+ public static Field lazyField = new Field(LAZY_FIELD_KEY, customType, LAZY_FIELD_TEXT);
public static final String LARGE_LAZY_FIELD_KEY = "largeLazyField";
public static String LARGE_LAZY_FIELD_TEXT;
@@ -102,15 +159,13 @@ class DocHelper {
//From Issue 509
public static final String FIELD_UTF1_TEXT = "field one \u4e00text";
public static final String TEXT_FIELD_UTF1_KEY = "textField1Utf8";
- public static Field textUtfField1 = new Field(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT,
- Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO);
+ public static Field textUtfField1 = new Field(TEXT_FIELD_UTF1_KEY, customType, FIELD_UTF1_TEXT);
public static final String FIELD_UTF2_TEXT = "field field field \u4e00two text";
//Fields will be lexicographically sorted. So, the order is: field, text, two
public static final int [] FIELD_UTF2_FREQS = {3, 1, 1};
public static final String TEXT_FIELD_UTF2_KEY = "textField2Utf8";
- public static Field textUtfField2 = new Field(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT, Field.Store.YES,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
+ public static Field textUtfField2 = new Field(TEXT_FIELD_UTF2_KEY, customType2, FIELD_UTF2_TEXT);
@@ -136,16 +191,16 @@ class DocHelper {
largeLazyField//placeholder for large field, since this is null. It must always be last
};
- public static Map<String,Fieldable> all =new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> indexed =new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> stored =new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> unstored=new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> unindexed=new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> termvector=new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> notermvector=new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> lazy= new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> noNorms=new HashMap<String,Fieldable>();
- public static Map<String,Fieldable> noTf=new HashMap<String,Fieldable>();
+ public static Map<String,IndexableField> all =new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> indexed =new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> stored =new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> unstored=new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> unindexed=new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> termvector=new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> notermvector=new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> lazy= new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> noNorms=new HashMap<String,IndexableField>();
+ public static Map<String,IndexableField> noTf=new HashMap<String,IndexableField>();
static {
//Initialize the large Lazy Field
@@ -159,28 +214,34 @@ class DocHelper {
LAZY_FIELD_BINARY_BYTES = "These are some binary field bytes".getBytes("UTF8");
} catch (UnsupportedEncodingException e) {
}
- lazyFieldBinary = new Field(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES);
+ lazyFieldBinary = new BinaryField(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES);
fields[fields.length - 2] = lazyFieldBinary;
LARGE_LAZY_FIELD_TEXT = buffer.toString();
- largeLazyField = new Field(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT, Field.Store.YES, Field.Index.ANALYZED);
+ largeLazyField = new Field(LARGE_LAZY_FIELD_KEY, customType, LARGE_LAZY_FIELD_TEXT);
fields[fields.length - 1] = largeLazyField;
for (int i=0; i<fields.length; i++) {
- Fieldable f = fields[i];
+ IndexableField f = fields[i];
add(all,f);
- if (f.isIndexed()) add(indexed,f);
+ if (f.indexed()) add(indexed,f);
else add(unindexed,f);
- if (f.isTermVectorStored()) add(termvector,f);
- if (f.isIndexed() && !f.isTermVectorStored()) add(notermvector,f);
- if (f.isStored()) add(stored,f);
+ if (f.storeTermVectors()) add(termvector,f);
+ if (f.indexed() && !f.storeTermVectors()) add(notermvector,f);
+ if (f.stored()) add(stored,f);
else add(unstored,f);
+<<<<<<<
if (f.getOmitNorms()) add(noNorms,f);
if (f.getIndexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f);
if (f.isLazy()) add(lazy, f);
+=======
+ if (f.omitNorms()) add(noNorms,f);
+ if (f.omitTermFreqAndPositions()) add(noTf,f);
+ //if (f.isLazy()) add(lazy, f);
+>>>>>>>
}
}
- private static void add(Map<String,Fieldable> map, Fieldable field) {
+ private static void add(Map<String,IndexableField> map, IndexableField field) {
map.put(field.name(), field);
}
@@ -249,6 +310,6 @@ class DocHelper {
}
public static int numFields(Document doc) {
- return doc.getFields().size();
+ return doc.size();
}
}
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Mon Aug 15 22:03:41 2011
@@ -24,8 +24,11 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+<<<<<<<
import org.apache.lucene.document.IndexDocValuesField;
import org.apache.lucene.document.Document;
+=======
+>>>>>>>
import org.apache.lucene.index.IndexWriter; // javadoc
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.values.ValueType;
@@ -120,23 +123,33 @@ public class RandomIndexWriter implement
/**
* Adds a Document.
- * @see IndexWriter#addDocument(Document)
+ * @see IndexWriter#addDocument(Iterable)
*/
+<<<<<<<
public void addDocument(final Document doc) throws IOException {
if (doDocValues) {
randomPerDocFieldValues(r, doc);
}
+=======
+ public <T extends IndexableField> void addDocument(final Iterable<T> doc) throws IOException {
+>>>>>>>
if (r.nextInt(5) == 3) {
// TODO: maybe, we should simply buffer up added docs
// (but we need to clone them), and only when
// getReader, commit, etc. are called, we do an
// addDocuments? Would be better testing.
- w.addDocuments(new Iterable<Document>() {
+ w.addDocuments(new Iterable<Iterable<T>>() {
+<<<<<<<
@Override
public Iterator<Document> iterator() {
return new Iterator<Document>() {
+=======
+ // @Override -- not until Java 1.6
+ public Iterator<Iterable<T>> iterator() {
+ return new Iterator<Iterable<T>>() {
+>>>>>>>
boolean done;
@Override
@@ -149,8 +162,13 @@ public class RandomIndexWriter implement
throw new UnsupportedOperationException();
}
+<<<<<<<
@Override
public Document next() {
+=======
+ // @Override -- not until Java 1.6
+ public Iterable<T> next() {
+>>>>>>>
if (done) {
throw new IllegalStateException();
}
@@ -238,31 +256,41 @@ public class RandomIndexWriter implement
}
}
- public void addDocuments(Iterable<Document> docs) throws IOException {
+ public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
w.addDocuments(docs);
maybeCommit();
}
- public void updateDocuments(Term delTerm, Iterable<Document> docs) throws IOException {
+ public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
w.updateDocuments(delTerm, docs);
maybeCommit();
}
/**
* Updates a document.
- * @see IndexWriter#updateDocument(Term, Document)
+ * @see IndexWriter#updateDocument(Term, Iterable)
*/
+<<<<<<<
public void updateDocument(final Term t, final Document doc) throws IOException {
if (doDocValues) {
randomPerDocFieldValues(r, doc);
}
+=======
+ public <T extends IndexableField> void updateDocument(Term t, final Iterable<T> doc) throws IOException {
+>>>>>>>
if (r.nextInt(5) == 3) {
- w.updateDocuments(t, new Iterable<Document>() {
+ w.updateDocuments(t, new Iterable<Iterable<T>>() {
+<<<<<<<
@Override
public Iterator<Document> iterator() {
return new Iterator<Document>() {
+=======
+ // @Override -- not until Java 1.6
+ public Iterator<Iterable<T>> iterator() {
+ return new Iterator<Iterable<T>>() {
+>>>>>>>
boolean done;
@Override
@@ -275,8 +303,13 @@ public class RandomIndexWriter implement
throw new UnsupportedOperationException();
}
+<<<<<<<
@Override
public Document next() {
+=======
+ // @Override -- not until Java 1.6
+ public Iterable<T> next() {
+>>>>>>>
if (done) {
throw new IllegalStateException();
}
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java Mon Aug 15 22:03:41 2011
@@ -30,6 +30,9 @@ import java.util.Random;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
/** Minimal port of contrib/benchmark's LneDocSource +
* DocMaker, so tests can enum docs from a line file created
@@ -117,19 +120,24 @@ public class LineFileDocs implements Clo
public DocState() {
doc = new Document();
- title = new Field("title", "", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS);
+ title = new StringField("title", "");
doc.add(title);
- titleTokenized = new Field("titleTokenized", "", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
+ FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(true);
+ ft.setStoreTermVectorOffsets(true);
+ ft.setStoreTermVectorPositions(true);
+
+ titleTokenized = new Field("titleTokenized", ft, "");
doc.add(titleTokenized);
- body = new Field("body", "", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
+ body = new Field("body", ft, "");
doc.add(body);
- id = new Field("docid", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
+ id = new Field("docid", StringField.TYPE_STORED, "");
doc.add(id);
- date = new Field("date", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
+ date = new Field("date", StringField.TYPE_STORED, "");
doc.add(date);
}
}
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Mon Aug 15 22:03:41 2011
@@ -36,10 +36,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.*;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
@@ -1108,12 +1105,11 @@ public abstract class LuceneTestCase ext
return dir;
}
- /** Returns a new field instance.
- * See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
- public static Field newField(String name, String value, Index index) {
- return newField(random, name, value, index);
+ public static org.apache.lucene.document.Field newField(String name, String value, FieldType type) {
+ return newField(random, name, value, type);
}
+<<<<<<<
/** Returns a new field instance.
* See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
public static Field newField(String name, String value, Store store, Index index) {
@@ -1149,44 +1145,53 @@ public abstract class LuceneTestCase ext
* See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
public static Field newField(Random random, String name, String value, Store store, Index index, TermVector tv) {
+=======
+ public static org.apache.lucene.document.Field newField(Random random, String name, String value, FieldType type) {
+>>>>>>>
if (usually(random)) {
// most of the time, don't modify the params
+<<<<<<<
return new Field(name, value, store, index, tv);
}
if (random.nextBoolean()) {
// tickle any code still relying on field names being interned:
name = new String(name);
+=======
+ return new org.apache.lucene.document.Field(name, type, value);
+>>>>>>>
}
- if (!index.isIndexed())
- return new Field(name, value, store, index, tv);
-
- if (!store.isStored() && random.nextBoolean())
- store = Store.YES; // randomly store it
-
- tv = randomTVSetting(random, tv);
+ FieldType newType = new FieldType(type);
+ if (!newType.stored() && random.nextBoolean()) {
+ newType.setStored(true); // randomly store it
+ }
- return new Field(name, value, store, index, tv);
- }
+ if (newType.indexed() && !newType.storeTermVectors()) {
+ newType.setStoreTermVectors(random.nextBoolean());
+ }
- static final TermVector tvSettings[] = {
- TermVector.NO, TermVector.YES, TermVector.WITH_OFFSETS,
- TermVector.WITH_POSITIONS, TermVector.WITH_POSITIONS_OFFSETS
- };
+ if (!newType.storeTermVectors()) {
+ if (!newType.storeTermVectorOffsets()) {
+ newType.setStoreTermVectorOffsets(random.nextBoolean());
+ }
+ if (!newType.storeTermVectorPositions()) {
+ newType.setStoreTermVectorPositions(random.nextBoolean());
+ }
+ }
- private static TermVector randomTVSetting(Random random, TermVector minimum) {
- switch(minimum) {
- case NO: return tvSettings[_TestUtil.nextInt(random, 0, tvSettings.length-1)];
- case YES: return tvSettings[_TestUtil.nextInt(random, 1, tvSettings.length-1)];
- case WITH_OFFSETS: return random.nextBoolean() ? TermVector.WITH_OFFSETS
- : TermVector.WITH_POSITIONS_OFFSETS;
- case WITH_POSITIONS: return random.nextBoolean() ? TermVector.WITH_POSITIONS
- : TermVector.WITH_POSITIONS_OFFSETS;
- default: return TermVector.WITH_POSITIONS_OFFSETS;
+ // TODO: we need to do this, but smarter, ie, most of
+ // the time we set the same value for a given field but
+ // sometimes (rarely) we change it up:
+ /*
+ if (newType.omitNorms()) {
+ newType.setOmitNorms(random.nextBoolean());
}
+ */
+
+ return new org.apache.lucene.document.Field(name, newType, value);
}
-
+
/** return a random Locale from the available locales on the system */
public static Locale randomLocale(Random random) {
Locale locales[] = Locale.getAvailableLocales();
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Mon Aug 15 22:03:41 2011
@@ -35,12 +35,16 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
import org.apache.lucene.document.Document;
+<<<<<<<
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
+=======
+>>>>>>>
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergeScheduler;
@@ -424,10 +428,16 @@ public class _TestUtil {
/** Adds field info for a Document. */
public static void add(Document doc, FieldInfos fieldInfos) {
+<<<<<<<
List<Fieldable> fields = doc.getFields();
for (Fieldable field : fields) {
fieldInfos.addOrUpdate(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(),
field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getIndexOptions(), field.docValuesType());
+=======
+ for (IndexableField field : doc) {
+ fieldInfos.addOrUpdate(field.name(), field.indexed(), field.storeTermVectors(), field.storeTermVectorPositions(),
+ field.storeTermVectorOffsets(), field.omitNorms(), false, field.omitTermFreqAndPositions());
+>>>>>>>
}
}
Modified: lucene/dev/branches/fieldtype_conflicted/lucene/src/test/org/apache/lucene/TestDemo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype_conflicted/lucene/src/test/org/apache/lucene/TestDemo.java?rev=1158029&r1=1158028&r2=1158029&view=diff
==============================================================================
--- lucene/dev/branches/fieldtype_conflicted/lucene/src/test/org/apache/lucene/TestDemo.java (original)
+++ lucene/dev/branches/fieldtype_conflicted/lucene/src/test/org/apache/lucene/TestDemo.java Mon Aug 15 22:03:41 2011
@@ -22,7 +22,8 @@ import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.*;
@@ -49,8 +50,9 @@ public class TestDemo extends LuceneTest
Document doc = new Document();
String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
String text = "This is the text to be indexed. " + longTerm;
- doc.add(newField("fieldname", text, Field.Store.YES,
- Field.Index.ANALYZED));
+ FieldType textType = new FieldType(TextField.TYPE_UNSTORED);
+ textType.setStored(true);
+ doc.add(newField("fieldname", text, textType));
iwriter.addDocument(doc);
iwriter.close();