You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/01/18 23:28:20 UTC
svn commit: r1233096 [7/13] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/modules/analysis/kuromoji/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/maven/modules/a...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java Wed Jan 18 22:28:07 2012
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
-import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
@@ -29,7 +28,6 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
-import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.search.SearcherManager; // javadocs
import org.apache.lucene.store.*;
import org.apache.lucene.util.Bits;
@@ -108,39 +106,6 @@ public abstract class IndexReader implem
}
}
- /**
- * Constants describing field properties, for example used for
- * {@link IndexReader#getFieldNames(FieldOption)}.
- */
- public static enum FieldOption {
- /** All fields */
- ALL,
- /** All indexed fields */
- INDEXED,
- /** All fields that store payloads */
- STORES_PAYLOADS,
- /** All fields that omit tf */
- OMIT_TERM_FREQ_AND_POSITIONS,
- /** All fields that omit positions */
- OMIT_POSITIONS,
- /** All fields which are not indexed */
- UNINDEXED,
- /** All fields which are indexed with termvectors enabled */
- INDEXED_WITH_TERMVECTOR,
- /** All fields which are indexed but don't have termvectors enabled */
- INDEXED_NO_TERMVECTOR,
- /** All fields with termvectors enabled. Please note that only standard termvector fields are returned */
- TERMVECTOR,
- /** All fields with termvectors with position values enabled */
- TERMVECTOR_WITH_POSITION,
- /** All fields with termvectors with offset values enabled */
- TERMVECTOR_WITH_OFFSET,
- /** All fields with termvectors with offset values and position values enabled */
- TERMVECTOR_WITH_POSITION_OFFSET,
- /** All fields holding doc values */
- DOC_VALUES
- }
-
private volatile boolean closed;
private final AtomicInteger refCount = new AtomicInteger();
@@ -823,9 +788,9 @@ public abstract class IndexReader implem
/** Returns {@link DocsAndPositionsEnum} for the specified
* field & term. This may return null, if either the
- * field or term does not exist, or, positions were not
- * indexed for this field. */
- public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
+ * field or term does not exist, or needsOffsets is
+ * true but offsets were not indexed for this field. */
+ public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, boolean needsOffsets) throws IOException {
assert field != null;
assert term != null;
final Fields fields = fields();
@@ -834,7 +799,7 @@ public abstract class IndexReader implem
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(term, true)) {
- return termsEnum.docsAndPositions(liveDocs, null);
+ return termsEnum.docsAndPositions(liveDocs, null, needsOffsets);
}
}
}
@@ -865,8 +830,9 @@ public abstract class IndexReader implem
* Returns {@link DocsAndPositionsEnum} for the specified field and
* {@link TermState}. This may return null, if either the field or the term
* does not exists, the {@link TermState} is invalid for the underlying
- * implementation, or positions were not indexed for this field. */
- public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
+ * implementation, or needsOffsets is true but offsets
+ * were not indexed for this field. */
+ public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state, boolean needsOffsets) throws IOException {
assert state != null;
assert field != null;
final Fields fields = fields();
@@ -875,7 +841,7 @@ public abstract class IndexReader implem
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
termsEnum.seekExact(term, state);
- return termsEnum.docsAndPositions(liveDocs, null);
+ return termsEnum.docsAndPositions(liveDocs, null, needsOffsets);
}
}
return null;
@@ -897,15 +863,14 @@ public abstract class IndexReader implem
/** Implements close. */
protected abstract void doClose() throws IOException;
-
/**
- * Get a list of unique field names that exist in this index and have the specified
- * field option information.
- * @param fldOption specifies which field option should be available for the returned fields
- * @return Collection of Strings indicating the names of the fields.
- * @see IndexReader.FieldOption
+ * Get the {@link FieldInfos} describing all fields in
+ * this reader. NOTE: do not make any changes to the
+ * returned FieldInfos!
+ *
+ * @lucene.experimental
*/
- public abstract Collection<String> getFieldNames(FieldOption fldOption);
+ public abstract FieldInfos getFieldInfos();
/** Returns the {@link Bits} representing live (not
* deleted) docs. A set bit indicates the doc ID has not
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed Jan 18 22:28:07 2012
@@ -3174,6 +3174,8 @@ public class IndexWriter implements Clos
if (dropSegment) {
readerPool.drop(merge.info);
+ deleter.deleteNewFiles(merge.info.files());
+ assert !segmentInfos.contains(merge.info);
} else {
if (mergedDeletes != null && !poolReaders) {
mergedDeletes.writeLiveDocs(directory);
@@ -3191,7 +3193,7 @@ public class IndexWriter implements Clos
closeMergeReaders(merge, false);
- if (merge.maxNumSegments != -1) {
+ if (merge.maxNumSegments != -1 && !dropSegment) {
// cascade the forceMerge:
if (!segmentsToMerge.containsKey(merge.info)) {
segmentsToMerge.put(merge.info, Boolean.FALSE);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java Wed Jan 18 22:28:07 2012
@@ -22,8 +22,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.NumericField;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.util.BytesRef;
// TODO: how to handle versioning here...?
@@ -37,17 +35,16 @@ import org.apache.lucene.util.BytesRef;
public interface IndexableField {
- // TODO: add attrs to this API?
-
/** Field name */
public String name();
- // NOTE: if doc/field impl has the notion of "doc level boost"
- // it must be multiplied in w/ this field's boost
-
+ /** {@link IndexableFieldType} describing the properties
+ * of this field. */
+ public IndexableFieldType fieldType();
+
/** Field boost (you must pre-multiply in any doc boost). */
public float boost();
-
+
/** Non-null if this field has a binary value */
public BytesRef binaryValue();
@@ -57,31 +54,10 @@ public interface IndexableField {
/** Non-null if this field has a Reader value */
public Reader readerValue();
- // Numeric field:
- /** True if this field is numeric */
- public boolean numeric();
-
- /** Numeric {@link org.apache.lucene.document.NumericField.DataType}; only used if
- * the field is numeric */
- public NumericField.DataType numericDataType();
-
- /** Numeric value; only used if the field is numeric */
+ /** Non-null if this field hasa numeric value */
public Number numericValue();
/**
- * Returns the IndexableFieldType describing the properties of this field
- *
- * @return IndexableFieldType for this field
- */
- public IndexableFieldType fieldType();
-
- /** Non-null if doc values should be indexed */
- public DocValue docValue();
-
- /** DocValues type; only used if docValue is non-null */
- public DocValues.Type docValueType();
-
- /**
* Creates the TokenStream used for indexing this field. If appropriate,
* implementations should use the given Analyzer to create the TokenStreams.
*
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java Wed Jan 18 22:28:07 2012
@@ -46,4 +46,8 @@ public interface IndexableFieldType {
/** {@link IndexOptions}, describing what should be
* recorded into the inverted index */
public IndexOptions indexOptions();
+
+ /** DocValues type; if non-null then the field's value
+ * will be indexed into docValues */
+ public DocValues.Type docValueType();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocValues.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocValues.java Wed Jan 18 22:28:07 2012
@@ -46,6 +46,13 @@ public class MultiDocValues extends DocV
public DocValues pull(IndexReader reader, String field) throws IOException {
return reader.normValues(field);
}
+
+ public boolean stopLoadingOnNull(IndexReader reader, String field) throws IOException {
+ // for norms we drop all norms if one leaf reader has no norms and the field is present
+ FieldInfos fieldInfos = reader.getFieldInfos();
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ return fieldInfo != null && fieldInfo.omitNorms;
+ }
};
public static class DocValuesSlice {
@@ -65,6 +72,10 @@ public class MultiDocValues extends DocV
public DocValues pull(IndexReader reader, String field) throws IOException {
return reader.docValues(field);
}
+
+ public boolean stopLoadingOnNull(IndexReader reader, String field) throws IOException {
+ return false;
+ }
}
private DocValuesSlice[] slices;
@@ -123,12 +134,19 @@ public class MultiDocValues extends DocV
// potentially incompatible types
new ReaderUtil.Gather(r) {
+ boolean stop = false;
@Override
protected void add(int base, IndexReader r) throws IOException {
+ if (stop) {
+ return;
+ }
final DocValues d = puller.pull(r, field);
if (d != null) {
TypePromoter incoming = TypePromoter.create(d.type(), d.getValueSize());
promotedType[0] = promotedType[0].promote(incoming);
+ } else if (puller.stopLoadingOnNull(r, field)){
+ promotedType[0] = TypePromoter.getIdentityPromoter(); // set to identity to return null
+ stop = true;
}
slices.add(new DocValuesSlice(d, base, r.maxDoc()));
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java Wed Jan 18 22:28:07 2012
@@ -126,6 +126,16 @@ public final class MultiDocsAndPositions
}
@Override
+ public int startOffset() throws IOException {
+ return current.startOffset();
+ }
+
+ @Override
+ public int endOffset() throws IOException {
+ return current.endOffset();
+ }
+
+ @Override
public boolean hasPayload() {
return current.hasPayload();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiFields.java Wed Jan 18 22:28:07 2012
@@ -167,14 +167,14 @@ public final class MultiFields extends F
/** Returns {@link DocsAndPositionsEnum} for the specified
* field & term. This may return null if the term does
* not exist or positions were not indexed. */
- public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException {
+ public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsOffsets) throws IOException {
assert field != null;
assert term != null;
final Terms terms = getTerms(r, field);
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(term, true)) {
- return termsEnum.docsAndPositions(liveDocs, null);
+ return termsEnum.docsAndPositions(liveDocs, null, needsOffsets);
}
}
return null;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java Wed Jan 18 22:28:07 2012
@@ -418,7 +418,7 @@ public final class MultiTermsEnum extend
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
MultiDocsAndPositionsEnum docsAndPositionsEnum;
// Can only reuse if incoming enum is also a MultiDocsAndPositionsEnum
if (reuse != null && reuse instanceof MultiDocsAndPositionsEnum) {
@@ -469,7 +469,7 @@ public final class MultiTermsEnum extend
}
assert entry.index < docsAndPositionsEnum.subDocsAndPositionsEnum.length: entry.index + " vs " + docsAndPositionsEnum.subDocsAndPositionsEnum.length + "; " + subs.length;
- final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index]);
+ final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], needsOffsets);
if (subPostings != null) {
docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index] = subPostings;
@@ -479,8 +479,8 @@ public final class MultiTermsEnum extend
} else {
if (entry.terms.docs(b, null, false) != null) {
// At least one of our subs does not store
- // positions -- we can't correctly produce a
- // MultiDocsAndPositions enum
+ // offsets or positions -- we can't correctly
+ // produce a MultiDocsAndPositions enum
return null;
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NormsConsumer.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NormsConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NormsConsumer.java Wed Jan 18 22:28:07 2012
@@ -24,9 +24,7 @@ import java.util.Map;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PerDocConsumer;
-import org.apache.lucene.document.DocValuesField;
import org.apache.lucene.index.DocValues.Type;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
// TODO FI: norms could actually be stored as doc store
@@ -39,11 +37,9 @@ import org.apache.lucene.util.IOUtils;
final class NormsConsumer extends InvertedDocEndConsumer {
private final NormsFormat normsFormat;
private PerDocConsumer consumer;
- private final DocumentsWriterPerThread dwpt;
public NormsConsumer(DocumentsWriterPerThread dwpt) {
normsFormat = dwpt.codec.normsFormat();
- this.dwpt = dwpt;
}
@Override
@@ -71,14 +67,12 @@ final class NormsConsumer extends Invert
if (!fi.omitNorms) {
if (toWrite != null && toWrite.initialized()) {
anythingFlushed = true;
- toWrite.flush(state.numDocs);
+ final Type type = toWrite.flush(state.numDocs);
+ assert fi.getNormType() == type;
} else if (fi.isIndexed) {
anythingFlushed = true;
- final DocValuesConsumer valuesConsumer = newConsumer(new PerDocWriteState(state), fi);
- final DocValuesField value = new DocValuesField("");
- value.setBytes(new BytesRef(new byte[] {0x00}), Type.BYTES_FIXED_STRAIGHT);
- valuesConsumer.add(state.numDocs-1, value);
- valuesConsumer.finish(state.numDocs);
+ assert fi.getNormType() == null;
+ fi.setNormValueType(null, false);
}
}
}
@@ -110,12 +104,12 @@ final class NormsConsumer extends Invert
}
DocValuesConsumer newConsumer(PerDocWriteState perDocWriteState,
- FieldInfo fieldInfo) throws IOException {
+ FieldInfo fieldInfo, Type type) throws IOException {
if (consumer == null) {
consumer = normsFormat.docsConsumer(perDocWriteState);
}
- DocValuesConsumer addValuesField = consumer.addValuesField(
- Type.BYTES_FIXED_STRAIGHT, fieldInfo);
+ DocValuesConsumer addValuesField = consumer.addValuesField(type, fieldInfo);
return addValuesField;
}
+
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java Wed Jan 18 22:28:07 2012
@@ -19,6 +19,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.document.DocValuesField;
+import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.BytesRef;
@@ -29,9 +30,9 @@ public class NormsConsumerPerField exten
private final Similarity similarity;
private final FieldInvertState fieldState;
private DocValuesConsumer consumer;
- private final DocValuesField value = new DocValuesField("");
- private final BytesRef spare = new BytesRef(1);
+ private final Norm norm;
private final NormsConsumer parent;
+ private Type initType;
public NormsConsumerPerField(final DocInverterPerField docInverterPerField, final FieldInfo fieldInfo, NormsConsumer parent) {
this.fieldInfo = fieldInfo;
@@ -39,10 +40,9 @@ public class NormsConsumerPerField exten
docState = docInverterPerField.docState;
fieldState = docInverterPerField.fieldState;
similarity = docState.similarityProvider.get(fieldInfo.name);
- spare.length = 1;
- spare.offset = 0;
-
+ norm = new Norm();
}
+
@Override
public int compareTo(NormsConsumerPerField other) {
return fieldInfo.name.compareTo(other.fieldInfo.name);
@@ -51,22 +51,33 @@ public class NormsConsumerPerField exten
@Override
void finish() throws IOException {
if (fieldInfo.isIndexed && !fieldInfo.omitNorms) {
- DocValuesConsumer consumer = getConsumer();
- spare.bytes[0] = similarity.computeNorm(fieldState);
- value.setBytes(spare, Type.BYTES_FIXED_STRAIGHT);
- consumer.add(docState.docID, value);
+ similarity.computeNorm(fieldState, norm);
+ if (norm.type() != null) {
+ IndexableField field = norm.field();
+ // some similarity might not compute any norms
+ DocValuesConsumer consumer = getConsumer(norm.type());
+ consumer.add(docState.docID, field);
+ }
}
}
- void flush(int docCount) throws IOException {
- assert initialized();
+ Type flush(int docCount) throws IOException {
+ if (!initialized()) {
+ return null; // null type - not omitted but not written
+ }
consumer.finish(docCount);
+ return initType;
}
- private DocValuesConsumer getConsumer() throws IOException {
+ private DocValuesConsumer getConsumer(Type type) throws IOException {
if (consumer == null) {
- consumer = parent.newConsumer(docState.docWriter.newPerDocWriteState(""), fieldInfo);
+ fieldInfo.setNormValueType(type, false);
+ consumer = parent.newConsumer(docState.docWriter.newPerDocWriteState(""), fieldInfo, type);
+ this.initType = type;
+ }
+ if (initType != type) {
+ throw new IllegalArgumentException("NormTypes for field: " + fieldInfo.name + " doesn't match " + initType + " != " + type);
}
return consumer;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java Wed Jan 18 22:28:07 2012
@@ -17,12 +17,13 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-
import java.io.IOException;
import java.util.*;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.ReaderUtil;
+
/** An IndexReader which reads multiple, parallel indexes. Each index added
* must have the same number of documents, but typically each contains
@@ -53,6 +54,7 @@ public class ParallelReader extends Inde
private int maxDoc;
private int numDocs;
private boolean hasDeletions;
+ private final FieldInfos fieldInfos;
private final ParallelFields fields = new ParallelFields();
@@ -68,6 +70,7 @@ public class ParallelReader extends Inde
public ParallelReader(boolean closeSubReaders) throws IOException {
super();
this.incRefReaders = !closeSubReaders;
+ fieldInfos = new FieldInfos();
}
/** {@inheritDoc} */
@@ -120,12 +123,13 @@ public class ParallelReader extends Inde
throw new IllegalArgumentException
("All readers must have same numDocs: "+numDocs+"!="+reader.numDocs());
- Collection<String> fields = reader.getFieldNames(IndexReader.FieldOption.ALL);
- readerToFields.put(reader, fields);
- for (final String field : fields) { // update fieldToReader map
- if (fieldToReader.get(field) == null) {
- fieldToReader.put(field, reader);
- this.fields.addField(field, MultiFields.getFields(reader).terms(field));
+ final FieldInfos readerFieldInfos = ReaderUtil.getMergedFieldInfos(reader);
+ for(FieldInfo fieldInfo : readerFieldInfos) { // update fieldToReader map
+ // NOTE: first reader having a given field "wins":
+ if (fieldToReader.get(fieldInfo.name) == null) {
+ fieldInfos.add(fieldInfo);
+ fieldToReader.put(fieldInfo.name, reader);
+ this.fields.addField(fieldInfo.name, MultiFields.getFields(reader).terms(fieldInfo.name));
}
}
@@ -192,6 +196,11 @@ public class ParallelReader extends Inde
return fields.size();
}
}
+
+ @Override
+ public FieldInfos getFieldInfos() {
+ return fieldInfos;
+ }
@Override
public Bits getLiveDocs() {
@@ -384,17 +393,6 @@ public class ParallelReader extends Inde
}
@Override
- public Collection<String> getFieldNames (IndexReader.FieldOption fieldNames) {
- ensureOpen();
- Set<String> fieldSet = new HashSet<String>();
- for (final IndexReader reader : readers) {
- Collection<String> names = reader.getFieldNames(fieldNames);
- fieldSet.addAll(names);
- }
- return fieldSet;
- }
-
- @Override
public ReaderContext getTopReaderContext() {
ensureOpen();
return topLevelReaderContext;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Wed Jan 18 22:28:07 2012
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -30,9 +29,6 @@ import org.apache.lucene.codecs.FieldsCo
import org.apache.lucene.codecs.PerDocConsumer;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.TermVectorsWriter;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.IndexReader.FieldOption;
-import org.apache.lucene.index.DocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
@@ -135,18 +131,6 @@ final class SegmentMerger {
return mergeState;
}
- private static void addIndexed(IndexReader reader, FieldInfos fInfos,
- Collection<String> names, boolean storeTermVectors,
- boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
- boolean storePayloads, IndexOptions indexOptions)
- throws IOException {
- for (String field : names) {
- fInfos.addOrUpdate(field, true, storeTermVectors,
- storePositionWithTermVector, storeOffsetWithTermVector, !reader
- .hasNorms(field), storePayloads, indexOptions, null);
- }
- }
-
private void setMatchingSegmentReaders() {
// If the i'th reader is a SegmentReader and has
// identical fieldName -> number mapping, then this
@@ -160,10 +144,15 @@ final class SegmentMerger {
// stored fields:
for (int i = 0; i < numReaders; i++) {
MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i);
+ // TODO: we may be able to broaden this to
+ // non-SegmentReaders, since FieldInfos is now
+ // required? But... this'd also require exposing
+ // bulk-copy (TVs and stored fields) API in foreign
+ // readers..
if (reader.reader instanceof SegmentReader) {
SegmentReader segmentReader = (SegmentReader) reader.reader;
boolean same = true;
- FieldInfos segmentFieldInfos = segmentReader.fieldInfos();
+ FieldInfos segmentFieldInfos = segmentReader.getFieldInfos();
for (FieldInfo fi : segmentFieldInfos) {
if (!mergeState.fieldInfos.fieldName(fi.number).equals(fi.name)) {
same = false;
@@ -202,65 +191,68 @@ final class SegmentMerger {
}
private void mergeFieldInfos() throws IOException {
+ mergeDocValuesAndNormsFieldInfos();
+ // write the merged infos
+ FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat()
+ .getFieldInfosWriter();
+ fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
+ }
+
+ public void mergeDocValuesAndNormsFieldInfos() throws IOException {
// mapping from all docvalues fields found to their promoted types
// this is because FieldInfos does not store the valueSize
Map<FieldInfo,TypePromoter> docValuesTypes = new HashMap<FieldInfo,TypePromoter>();
+ Map<FieldInfo,TypePromoter> normValuesTypes = new HashMap<FieldInfo,TypePromoter>();
for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) {
final IndexReader reader = readerAndLiveDocs.reader;
- if (reader instanceof SegmentReader) {
- SegmentReader segmentReader = (SegmentReader) reader;
- FieldInfos readerFieldInfos = segmentReader.fieldInfos();
- for (FieldInfo fi : readerFieldInfos) {
- FieldInfo merged = mergeState.fieldInfos.add(fi);
- // update the type promotion mapping for this reader
- if (fi.hasDocValues()) {
- TypePromoter previous = docValuesTypes.get(merged);
- docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
- }
- }
- } else {
- addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR), true, false, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.OMIT_POSITIONS), false, false, false, false, IndexOptions.DOCS_AND_FREQS);
- addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.OMIT_TERM_FREQ_AND_POSITIONS), false, false, false, false, IndexOptions.DOCS_ONLY);
- addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.INDEXED), false, false, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- mergeState.fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.UNINDEXED), false);
- Collection<String> dvNames = reader.getFieldNames(FieldOption.DOC_VALUES);
- mergeState.fieldInfos.addOrUpdate(dvNames, false);
- for (String dvName : dvNames) {
- FieldInfo merged = mergeState.fieldInfos.fieldInfo(dvName);
- DocValues docValues = reader.docValues(dvName);
- merged.setDocValuesType(docValues.type());
+ FieldInfos readerFieldInfos = reader.getFieldInfos();
+ for (FieldInfo fi : readerFieldInfos) {
+ FieldInfo merged = mergeState.fieldInfos.add(fi);
+ // update the type promotion mapping for this reader
+ if (fi.hasDocValues()) {
TypePromoter previous = docValuesTypes.get(merged);
- docValuesTypes.put(merged, mergeDocValuesType(previous, docValues));
+ docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
+ }
+ if (fi.normsPresent()) {
+ TypePromoter previous = normValuesTypes.get(merged);
+ normValuesTypes.put(merged, mergeDocValuesType(previous, reader.normValues(fi.name)));
}
}
}
-
+ updatePromoted(normValuesTypes, true);
+ updatePromoted(docValuesTypes, false);
+ }
+
+ protected void updatePromoted(Map<FieldInfo,TypePromoter> infoAndPromoter, boolean norms) {
// update any promoted doc values types:
- for (Map.Entry<FieldInfo,TypePromoter> e : docValuesTypes.entrySet()) {
+ for (Map.Entry<FieldInfo,TypePromoter> e : infoAndPromoter.entrySet()) {
FieldInfo fi = e.getKey();
TypePromoter promoter = e.getValue();
if (promoter == null) {
- fi.resetDocValuesType(null);
+ if (norms) {
+ fi.setNormValueType(null, true);
+ } else {
+ fi.setDocValuesType(null, true);
+ }
} else {
assert promoter != TypePromoter.getIdentityPromoter();
- if (fi.getDocValuesType() != promoter.type()) {
- // reset the type if we got promoted
- fi.resetDocValuesType(promoter.type());
+ if (norms) {
+ if (fi.getNormType() != promoter.type()) {
+ // reset the type if we got promoted
+ fi.setNormValueType(promoter.type(), true);
+ }
+ } else {
+ if (fi.getDocValuesType() != promoter.type()) {
+ // reset the type if we got promoted
+ fi.setDocValuesType(promoter.type(), true);
+ }
}
}
}
-
- // write the merged infos
- FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
- fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
}
+
/**
*
* @return The number of documents in all of the readers
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentReader.java Wed Jan 18 22:28:07 2012
@@ -18,16 +18,11 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Set;
import org.apache.lucene.store.Directory;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
-import org.apache.lucene.index.DocValues.Source;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
@@ -162,7 +157,8 @@ public final class SegmentReader extends
return liveDocs != null;
}
- FieldInfos fieldInfos() {
+ @Override
+ public FieldInfos getFieldInfos() {
return core.fieldInfos;
}
@@ -198,67 +194,11 @@ public final class SegmentReader extends
return si.docCount;
}
- /**
- * @see IndexReader#getFieldNames(org.apache.lucene.index.IndexReader.FieldOption)
- */
- @Override
- public Collection<String> getFieldNames(IndexReader.FieldOption fieldOption) {
- ensureOpen();
-
- Set<String> fieldSet = new HashSet<String>();
- for (FieldInfo fi : core.fieldInfos) {
- if (fieldOption == IndexReader.FieldOption.ALL) {
- fieldSet.add(fi.name);
- }
- else if (!fi.isIndexed && fieldOption == IndexReader.FieldOption.UNINDEXED) {
- fieldSet.add(fi.name);
- }
- else if (fi.indexOptions == IndexOptions.DOCS_ONLY && fieldOption == IndexReader.FieldOption.OMIT_TERM_FREQ_AND_POSITIONS) {
- fieldSet.add(fi.name);
- }
- else if (fi.indexOptions == IndexOptions.DOCS_AND_FREQS && fieldOption == IndexReader.FieldOption.OMIT_POSITIONS) {
- fieldSet.add(fi.name);
- }
- else if (fi.storePayloads && fieldOption == IndexReader.FieldOption.STORES_PAYLOADS) {
- fieldSet.add(fi.name);
- }
- else if (fi.isIndexed && fieldOption == IndexReader.FieldOption.INDEXED) {
- fieldSet.add(fi.name);
- }
- else if (fi.isIndexed && fi.storeTermVector == false && fieldOption == IndexReader.FieldOption.INDEXED_NO_TERMVECTOR) {
- fieldSet.add(fi.name);
- }
- else if (fi.storeTermVector == true &&
- fi.storePositionWithTermVector == false &&
- fi.storeOffsetWithTermVector == false &&
- fieldOption == IndexReader.FieldOption.TERMVECTOR) {
- fieldSet.add(fi.name);
- }
- else if (fi.isIndexed && fi.storeTermVector && fieldOption == IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR) {
- fieldSet.add(fi.name);
- }
- else if (fi.storePositionWithTermVector && fi.storeOffsetWithTermVector == false && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION) {
- fieldSet.add(fi.name);
- }
- else if (fi.storeOffsetWithTermVector && fi.storePositionWithTermVector == false && fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_OFFSET) {
- fieldSet.add(fi.name);
- }
- else if ((fi.storeOffsetWithTermVector && fi.storePositionWithTermVector) &&
- fieldOption == IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET) {
- fieldSet.add(fi.name);
- }
- else if (fi.hasDocValues() && fieldOption == IndexReader.FieldOption.DOC_VALUES) {
- fieldSet.add(fi.name);
- }
- }
- return fieldSet;
- }
-
@Override
public boolean hasNorms(String field) {
ensureOpen();
FieldInfo fi = core.fieldInfos.fieldInfo(field);
- return fi != null && fi.isIndexed && !fi.omitNorms;
+ return fi.normsPresent();
}
/** @lucene.internal */
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java Wed Jan 18 22:28:07 2012
@@ -103,4 +103,9 @@ public final class SlowMultiReaderWrappe
ensureOpen();
return readerContext;
}
+
+ @Override
+ public FieldInfos getFieldInfos() {
+ return ReaderUtil.getMergedFieldInfos(in);
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Term.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Term.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Term.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/Term.java Wed Jan 18 22:28:07 2012
@@ -44,7 +44,7 @@ public final class Term implements Compa
field = fld;
this.bytes = bytes;
}
-
+
/** Constructs a Term with the given field and text.
* <p>Note that a null field or null text value results in undefined
* behavior for most Lucene APIs that accept a Term parameter. */
@@ -132,4 +132,8 @@ public final class Term implements Compa
@Override
public final String toString() { return field + ":" + bytes.utf8ToString(); }
+
+ public Term deepCopyOf() {
+ return new Term(field, BytesRef.deepCopyOf(bytes));
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Wed Jan 18 22:28:07 2012
@@ -38,7 +38,7 @@ final class TermVectorsConsumerPerField
boolean doVectorOffsets;
int maxNumPostings;
- OffsetAttribute offsetAttribute = null;
+ OffsetAttribute offsetAttribute;
public TermVectorsConsumerPerField(TermsHashPerField termsHashPerField, TermVectorsConsumer termsWriter, FieldInfo fieldInfo) {
this.termsHashPerField = termsHashPerField;
@@ -118,9 +118,7 @@ final class TermVectorsConsumerPerField
TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
final TermVectorsWriter tv = termsWriter.writer;
- // TODO: we may want to make this sort in same order
- // as Codec's terms dict?
- final int[] termIDs = termsHashPerField.sortPostings(BytesRef.getUTF8SortedAsUnicodeComparator());
+ final int[] termIDs = termsHashPerField.sortPostings(tv.getComparator());
tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets);
@@ -151,7 +149,7 @@ final class TermVectorsConsumerPerField
termsHashPerField.reset();
// commit the termVectors once successful success - FI will otherwise reset them
- fieldInfo.setStoreTermVectors(doVectorPositions, doVectorOffsets);
+ fieldInfo.setStoreTermVectors();
}
void shrinkHash() {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsEnum.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsEnum.java Wed Jan 18 22:28:07 2012
@@ -160,12 +160,13 @@ public abstract class TermsEnum {
/** Get {@link DocsAndPositionsEnum} for the current term.
* Do not call this when the enum is unpositioned.
- * This method will only return null if positions were
- * not indexed into the postings by this codec.
+ * This method will only return null if needsOffsets is
+ * true but offsets were not indexed.
* @param liveDocs unset bits are documents that should not
* be returned
- * @param reuse pass a prior DocsAndPositionsEnum for possible reuse */
- public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException;
+ * @param reuse pass a prior DocsAndPositionsEnum for possible reuse
+ * @param needsOffsets true if offsets are required */
+ public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException;
/**
* Expert: Returns the TermsEnums internal state to position the TermsEnum
@@ -238,7 +239,7 @@ public abstract class TermsEnum {
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) {
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) {
throw new IllegalStateException("this method should never be called");
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java Wed Jan 18 22:28:07 2012
@@ -293,7 +293,7 @@ final class TermsHashPerField extends In
@Override
public int[] init() {
- if(perField.postingsArray == null) {
+ if (perField.postingsArray == null) {
perField.postingsArray = perField.consumer.createPostingsArray(2);
bytesUsed.addAndGet(perField.postingsArray.size * perField.postingsArray.bytesPerPosting());
}
@@ -305,8 +305,7 @@ final class TermsHashPerField extends In
ParallelPostingsArray postingsArray = perField.postingsArray;
final int oldSize = perField.postingsArray.size;
postingsArray = perField.postingsArray = postingsArray.grow();
- bytesUsed
- .addAndGet((postingsArray.bytesPerPosting() * (postingsArray.size - oldSize)));
+ bytesUsed.addAndGet((postingsArray.bytesPerPosting() * (postingsArray.size - oldSize)));
return postingsArray.textStarts;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Wed Jan 18 22:28:07 2012
@@ -30,7 +30,6 @@ import org.apache.lucene.index.DocTermOr
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.TermState;
@@ -162,12 +161,27 @@ class FieldCacheImpl implements FieldCac
FieldCacheImpl.this.purge(owner);
}
};
+
+ private void initReader(IndexReader reader) {
+ if (reader instanceof SegmentReader) {
+ ((SegmentReader) reader).addCoreClosedListener(purgeCore);
+ } else if (reader.getSequentialSubReaders() != null) {
+ throw new UnsupportedOperationException("Please use SlowMultiReaderWrapper, if you really need a top level FieldCache");
+ } else {
+ // we have a slow reader of some sort, try to register a purge event
+ // rather than relying on gc:
+ Object key = reader.getCoreCacheKey();
+ if (key instanceof IndexReader) {
+ ((IndexReader)key).addReaderClosedListener(purgeReader);
+ } else {
+ // last chance
+ reader.addReaderClosedListener(purgeReader);
+ }
+ }
+ }
/** Expert: Internal cache. */
abstract static class Cache {
- Cache() {
- this.wrapper = null;
- }
Cache(FieldCacheImpl wrapper) {
this.wrapper = wrapper;
@@ -198,11 +212,7 @@ class FieldCacheImpl implements FieldCac
// First time this reader is using FieldCache
innerCache = new HashMap<Entry,Object>();
readerCache.put(readerKey, innerCache);
- if (reader instanceof SegmentReader) {
- ((SegmentReader) reader).addCoreClosedListener(wrapper.purgeCore);
- } else {
- reader.addReaderClosedListener(wrapper.purgeReader);
- }
+ wrapper.initReader(reader);
}
if (innerCache.get(key) == null) {
innerCache.put(key, value);
@@ -223,11 +233,7 @@ class FieldCacheImpl implements FieldCac
// First time this reader is using FieldCache
innerCache = new HashMap<Entry,Object>();
readerCache.put(readerKey, innerCache);
- if (reader instanceof SegmentReader) {
- ((SegmentReader) reader).addCoreClosedListener(wrapper.purgeCore);
- } else {
- reader.addReaderClosedListener(wrapper.purgeReader);
- }
+ wrapper.initReader(reader);
value = null;
} else {
value = innerCache.get(key);
@@ -339,7 +345,7 @@ class FieldCacheImpl implements FieldCac
}
final int maxDoc = reader.maxDoc();
final byte[] retArray = new byte[maxDoc];
- Terms terms = MultiFields.getTerms(reader, field);
+ Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@@ -412,7 +418,7 @@ class FieldCacheImpl implements FieldCac
}
final int maxDoc = reader.maxDoc();
final short[] retArray = new short[maxDoc];
- Terms terms = MultiFields.getTerms(reader, field);
+ Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@@ -511,7 +517,7 @@ class FieldCacheImpl implements FieldCac
final int maxDoc = reader.maxDoc();
int[] retArray = null;
- Terms terms = MultiFields.getTerms(reader, field);
+ Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@@ -583,7 +589,7 @@ class FieldCacheImpl implements FieldCac
throws IOException {
final String field = entryKey.field;
FixedBitSet res = null;
- Terms terms = MultiFields.getTerms(reader, field);
+ Terms terms = reader.terms(field);
final int maxDoc = reader.maxDoc();
if (terms != null) {
final int termsDocCount = terms.getDocCount();
@@ -661,7 +667,7 @@ class FieldCacheImpl implements FieldCac
final int maxDoc = reader.maxDoc();
float[] retArray = null;
- Terms terms = MultiFields.getTerms(reader, field);
+ Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@@ -749,7 +755,7 @@ class FieldCacheImpl implements FieldCac
final int maxDoc = reader.maxDoc();
long[] retArray = null;
- Terms terms = MultiFields.getTerms(reader, field);
+ Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@@ -838,7 +844,7 @@ class FieldCacheImpl implements FieldCac
final int maxDoc = reader.maxDoc();
double[] retArray = null;
- Terms terms = MultiFields.getTerms(reader, field);
+ Terms terms = reader.terms(field);
FixedBitSet docsWithField = null;
if (terms != null) {
if (setDocsWithField) {
@@ -1043,7 +1049,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
throw new UnsupportedOperationException();
}
@@ -1086,7 +1092,7 @@ class FieldCacheImpl implements FieldCac
protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField /* ignored */)
throws IOException {
- Terms terms = MultiFields.getTerms(reader, entryKey.field);
+ Terms terms = reader.terms(entryKey.field);
final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue();
@@ -1231,7 +1237,7 @@ class FieldCacheImpl implements FieldCac
protected Object createValue(IndexReader reader, Entry entryKey, boolean setDocsWithField /* ignored */)
throws IOException {
- Terms terms = MultiFields.getTerms(reader, entryKey.field);
+ Terms terms = reader.terms(entryKey.field);
final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue();
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Wed Jan 18 22:28:07 2012
@@ -272,8 +272,8 @@ public final class FuzzyTermsEnum extend
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs,
- DocsAndPositionsEnum reuse) throws IOException {
- return actualEnum.docsAndPositions(liveDocs, reuse);
+ DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+ return actualEnum.docsAndPositions(liveDocs, reuse, needsOffsets);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Wed Jan 18 22:28:07 2012
@@ -225,7 +225,7 @@ public class MultiPhraseQuery extends Qu
return null;
}
termsEnum.seekExact(term.bytes(), termState);
- postingsEnum = termsEnum.docsAndPositions(liveDocs, null);
+ postingsEnum = termsEnum.docsAndPositions(liveDocs, null, false);
if (postingsEnum == null) {
// term does exist, but has no positions
@@ -475,7 +475,7 @@ class UnionDocsAndPositionsEnum extends
continue;
}
termsEnum.seekExact(term.bytes(), termState);
- DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null);
+ DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, false);
if (postings == null) {
// term does exist, but has no positions
throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")");
@@ -528,6 +528,16 @@ class UnionDocsAndPositionsEnum extends
}
@Override
+ public int startOffset() {
+ return -1;
+ }
+
+ @Override
+ public int endOffset() {
+ return -1;
+ }
+
+ @Override
public BytesRef getPayload() {
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Wed Jan 18 22:28:07 2012
@@ -239,7 +239,7 @@ public class PhraseQuery extends Query {
return null;
}
te.seekExact(t.bytes(), state);
- DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null);
+ DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null, false);
// PhraseQuery on a field that did not index
// positions.
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java Wed Jan 18 22:28:07 2012
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -122,10 +123,11 @@ public class BM25Similarity extends Simi
}
}
+
@Override
- public final byte computeNorm(FieldInvertState state) {
+ public final void computeNorm(FieldInvertState state, Norm norm) {
final int numTerms = discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength();
- return encodeNormValue(state.getBoost(), numTerms);
+ norm.setByte(encodeNormValue(state.getBoost(), numTerms));
}
public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java Wed Jan 18 22:28:07 2012
@@ -1,6 +1,7 @@
package org.apache.lucene.search.similarities;
import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.util.BytesRef;
/**
@@ -22,7 +23,7 @@ import org.apache.lucene.util.BytesRef;
/** Expert: Default scoring implementation. */
public class DefaultSimilarity extends TFIDFSimilarity {
-
+
/** Implemented as
* <code>state.getBoost()*lengthNorm(numTerms)</code>, where
* <code>numTerms</code> is {@link FieldInvertState#getLength()} if {@link
@@ -32,13 +33,13 @@ public class DefaultSimilarity extends T
*
* @lucene.experimental */
@Override
- public byte computeNorm(FieldInvertState state) {
+ public void computeNorm(FieldInvertState state, Norm norm) {
final int numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
+ norm.setByte(encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)))));
}
/** Implemented as <code>sqrt(freq)</code>. */
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java Wed Jan 18 22:28:07 2012
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -40,8 +41,8 @@ public class MultiSimilarity extends Sim
}
@Override
- public byte computeNorm(FieldInvertState state) {
- return sims[0].computeNorm(state);
+ public void computeNorm(FieldInvertState state, Norm norm) {
+ sims[0].computeNorm(state, norm);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java Wed Jan 18 22:28:07 2012
@@ -24,6 +24,7 @@ import org.apache.lucene.document.DocVal
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader; // javadoc
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.index.Terms; // javadoc
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CollectionStatistics;
@@ -36,7 +37,6 @@ import org.apache.lucene.search.TermStat
import org.apache.lucene.search.spans.SpanQuery; // javadoc
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SmallFloat; // javadoc
-import org.apache.lucene.util.TermContext;
/**
@@ -55,8 +55,8 @@ import org.apache.lucene.util.TermContex
* <a href="#querytime">query-time</a>.
* <p>
* <a name="indextime"/>
- * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing
- * the Similarity implementation to return a per-document byte for the field that will
+ * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState, Norm)}, allowing
+ * the Similarity implementation to set a per-document value for the field that will
* be later accessible via {@link IndexReader#normValues(String)}. Lucene makes no assumption
* about what is in this byte, but it is most useful for encoding length normalization
* information.
@@ -109,23 +109,24 @@ import org.apache.lucene.util.TermContex
* @lucene.experimental
*/
public abstract class Similarity {
+
/**
* Computes the normalization value for a field, given the accumulated
* state of term processing for this field (see {@link FieldInvertState}).
*
- * <p>Implementations should calculate a byte value based on the field
- * state and then return that value.
+ * <p>Implementations should calculate a norm value based on the field
+ * state and set that value to the given {@link Norm}.
*
* <p>Matches in longer fields are less precise, so implementations of this
- * method usually return smaller values when <code>state.getLength()</code> is large,
+ * method usually set smaller values when <code>state.getLength()</code> is large,
* and larger values when <code>state.getLength()</code> is small.
*
* @lucene.experimental
*
* @param state current processing state for this field
- * @return the calculated byte norm
+ * @param norm holds the computed norm value when this method returns
*/
- public abstract byte computeNorm(FieldInvertState state);
+ public abstract void computeNorm(FieldInvertState state, Norm norm);
/**
* Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query.
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java Wed Jan 18 22:28:07 2012
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Norm;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TermStatistics;
@@ -234,13 +235,13 @@ public abstract class SimilarityBase ext
/** Encodes the document length in the same way as {@link TFIDFSimilarity}. */
@Override
- public byte computeNorm(FieldInvertState state) {
+ public void computeNorm(FieldInvertState state, Norm norm) {
final float numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength() / state.getBoost();
- return encodeNormValue(state.getBoost(), numTerms);
+ norm.setByte(encodeNormValue(state.getBoost(), numTerms));
}
/** Decodes a normalization factor (document length) stored in an index.
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java Wed Jan 18 22:28:07 2012
@@ -22,14 +22,12 @@ import java.io.IOException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.index.Term;
import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.SmallFloat;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/package.html?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/package.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/package.html Wed Jan 18 22:28:07 2012
@@ -155,7 +155,7 @@ subclassing the Similarity, one can simp
matching term occurs. In these
cases people have overridden Similarity to return 1 from the tf() method.</p></li>
<li><p>Changing Length Normalization — By overriding
- {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState state)},
+ {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState state, Norm)},
it is possible to discount how the length of a field contributes
to a score. In {@link org.apache.lucene.search.similarities.DefaultSimilarity},
lengthNorm = 1 / (numTerms in field)^0.5, but if one changes this to be
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Wed Jan 18 22:28:07 2012
@@ -120,7 +120,7 @@ public class SpanTermQuery extends SpanQ
final TermsEnum termsEnum = context.reader.terms(term.field()).iterator(null);
termsEnum.seekExact(term.bytes(), state);
- final DocsAndPositionsEnum postings = termsEnum.docsAndPositions(acceptDocs, null);
+ final DocsAndPositionsEnum postings = termsEnum.docsAndPositions(acceptDocs, null, false);
if (postings != null) {
return new TermSpans(postings, term);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java Wed Jan 18 22:28:07 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.EOFException;
import java.io.IOException;
/** Base implementation class for buffered {@link IndexInput}. */
@@ -138,7 +139,7 @@ public abstract class BufferedIndexInput
if(bufferLength<len){
// Throw an exception when refill() could not read len bytes:
System.arraycopy(buffer, 0, b, offset, bufferLength);
- throw new IOException("read past EOF");
+ throw new EOFException("read past EOF: " + this);
} else {
System.arraycopy(buffer, 0, b, offset, len);
bufferPosition=len;
@@ -153,7 +154,7 @@ public abstract class BufferedIndexInput
// had in the buffer.
long after = bufferStart+bufferPosition+len;
if(after > length())
- throw new IOException("read past EOF");
+ throw new EOFException("read past EOF: " + this);
readInternal(b, offset, len);
bufferStart = after;
bufferPosition = 0;
@@ -231,7 +232,7 @@ public abstract class BufferedIndexInput
end = length();
int newLength = (int)(end - start);
if (newLength <= 0)
- throw new IOException("read past EOF");
+ throw new EOFException("read past EOF: " + this);
if (buffer == null) {
newBuffer(new byte[bufferSize]); // allocate buffer lazily
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/ByteArrayDataInput.java Wed Jan 18 22:28:07 2012
@@ -100,11 +100,11 @@ public final class ByteArrayDataInput ex
@Override
public int readVInt() {
- checkBounds();
+ assert checkBounds();
byte b = bytes[pos++];
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- checkBounds();
+ assert checkBounds();
b = bytes[pos++];
i |= (b & 0x7F) << shift;
}
@@ -113,11 +113,11 @@ public final class ByteArrayDataInput ex
@Override
public long readVLong() {
- checkBounds();
+ assert checkBounds();
byte b = bytes[pos++];
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- checkBounds();
+ assert checkBounds();
b = bytes[pos++];
i |= (b & 0x7FL) << shift;
}
@@ -127,7 +127,7 @@ public final class ByteArrayDataInput ex
// NOTE: AIOOBE not EOF if you read too much
@Override
public byte readByte() {
- checkBounds();
+ assert checkBounds();
return bytes[pos++];
}
@@ -140,7 +140,6 @@ public final class ByteArrayDataInput ex
}
private boolean checkBounds() {
- assert pos < limit;
- return true;
+ return pos < limit;
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java Wed Jan 18 22:28:07 2012
@@ -216,7 +216,7 @@ public final class CompoundFileDirectory
final String id = IndexFileNames.stripSegmentName(name);
final FileEntry entry = entries.get(id);
if (entry == null) {
- throw new IOException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")");
+ throw new FileNotFoundException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")");
}
return handle.openSlice(name, entry.offset, entry.length);
}
@@ -310,7 +310,7 @@ public final class CompoundFileDirectory
final String id = IndexFileNames.stripSegmentName(name);
final FileEntry entry = entries.get(id);
if (entry == null) {
- throw new IOException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")");
+ throw new FileNotFoundException("No sub-file with id " + id + " found (fileName=" + name + " files: " + entries.keySet() + ")");
}
return new IndexInputSlicer() {
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/DataInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/DataInput.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/DataInput.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/DataInput.java Wed Jan 18 22:28:07 2012
@@ -21,6 +21,8 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.util.IOUtils;
+
/**
* Abstract base class for performing read operations of Lucene's low-level
* data types.
@@ -166,7 +168,7 @@ public abstract class DataInput implemen
int length = readVInt();
final byte[] bytes = new byte[length];
readBytes(bytes, 0, length);
- return new String(bytes, 0, length, "UTF-8");
+ return new String(bytes, 0, length, IOUtils.CHARSET_UTF_8);
}
/** Returns a clone of this stream.
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/Directory.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/Directory.java Wed Jan 18 22:28:07 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Closeable;
@@ -305,7 +306,7 @@ public abstract class Directory implemen
protected void readInternal(byte[] b, int offset, int len) throws IOException {
long start = getFilePointer();
if(start + len > length)
- throw new IOException("read past EOF");
+ throw new EOFException("read past EOF: " + this);
base.seek(fileOffset + start);
base.readBytes(b, offset, len, false);
}
@@ -338,7 +339,7 @@ public abstract class Directory implemen
if (numBytes > 0) {
long start = getFilePointer();
if (start + numBytes > length) {
- throw new IOException("read past EOF");
+ throw new EOFException("read past EOF: " + this);
}
base.seek(fileOffset + start);
base.copyBytes(out, numBytes);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Wed Jan 18 22:28:07 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.EOFException;
import java.io.IOException;
import java.io.File;
import java.io.RandomAccessFile;
@@ -303,7 +304,7 @@ public class MMapDirectory extends FSDir
do {
curBufIndex++;
if (curBufIndex >= buffers.length) {
- throw new IOException("read past EOF: " + this);
+ throw new EOFException("read past EOF: " + this);
}
curBuf = buffers[curBufIndex];
curBuf.position(0);
@@ -326,7 +327,7 @@ public class MMapDirectory extends FSDir
offset += curAvail;
curBufIndex++;
if (curBufIndex >= buffers.length) {
- throw new IOException("read past EOF: " + this);
+ throw new EOFException("read past EOF: " + this);
}
curBuf = buffers[curBufIndex];
curBuf.position(0);
@@ -394,12 +395,12 @@ public class MMapDirectory extends FSDir
if (pos < 0L) {
throw new IllegalArgumentException("Seeking to negative position: " + this);
}
- throw new IOException("seek past EOF");
+ throw new EOFException("seek past EOF: " + this);
} catch (IllegalArgumentException iae) {
if (pos < 0L) {
throw new IllegalArgumentException("Seeking to negative position: " + this);
}
- throw new IOException("seek past EOF: " + this);
+ throw new EOFException("seek past EOF: " + this);
} catch (NullPointerException npe) {
throw new AlreadyClosedException("MMapIndexInput already closed: " + this);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java Wed Jan 18 22:28:07 2012
@@ -110,9 +110,6 @@ public class NIOFSDirectory extends FSDi
private ByteBuffer byteBuf; // wraps the buffer for NIO
- private byte[] otherBuffer;
- private ByteBuffer otherByteBuf;
-
final FileChannel channel;
public NIOFSIndexInput(File path, IOContext context, int chunkSize) throws IOException {
@@ -157,22 +154,7 @@ public class NIOFSDirectory extends FSDi
byteBuf.limit(len);
bb = byteBuf;
} else {
- if (offset == 0) {
- if (otherBuffer != b) {
- // Now wrap this other buffer; with compound
- // file, we are repeatedly called with its
- // buffer, so we wrap it once and then re-use it
- // on subsequent calls
- otherBuffer = b;
- otherByteBuf = ByteBuffer.wrap(b);
- } else
- otherByteBuf.clear();
- otherByteBuf.limit(len);
- bb = otherByteBuf;
- } else {
- // Always wrap when offset != 0
- bb = ByteBuffer.wrap(b, offset, len);
- }
+ bb = ByteBuffer.wrap(b, offset, len);
}
int readOffset = bb.position();
@@ -182,7 +164,7 @@ public class NIOFSDirectory extends FSDi
long pos = getFilePointer() + off;
if (pos + len > end) {
- throw new EOFException("read past EOF (resource: " + this + ")");
+ throw new EOFException("read past EOF: " + this);
}
try {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/RAMInputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/RAMInputStream.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/RAMInputStream.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/RAMInputStream.java Wed Jan 18 22:28:07 2012
@@ -91,7 +91,7 @@ public class RAMInputStream extends Inde
if (currentBufferIndex >= file.numBuffers()) {
// end of file reached, no more buffers left
if (enforceEOF) {
- throw new EOFException("Read past EOF (resource: " + this + ")");
+ throw new EOFException("read past EOF: " + this);
} else {
// Force EOF if a read takes place at this position
currentBufferIndex--;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java Wed Jan 18 22:28:07 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.store;
* limitations under the License.
*/
+import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
@@ -145,7 +146,7 @@ public class SimpleFSDirectory extends F
int total = 0;
if (position + len > end) {
- throw new IOException("read past EOF: " + this);
+ throw new EOFException("read past EOF: " + this);
}
try {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/ReaderUtil.java?rev=1233096&r1=1233095&r2=1233096&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/ReaderUtil.java Wed Jan 18 22:28:07 2012
@@ -18,9 +18,13 @@ package org.apache.lucene.util;
*/
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.io.IOException;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.CompositeReaderContext;
@@ -292,4 +296,26 @@ public final class ReaderUtil {
}
return hi;
}
+
+ public static Collection<String> getIndexedFields(IndexReader reader) {
+ final Collection<String> fields = new HashSet<String>();
+ for(FieldInfo fieldInfo : getMergedFieldInfos(reader)) {
+ if (fieldInfo.isIndexed) {
+ fields.add(fieldInfo.name);
+ }
+ }
+ return fields;
+ }
+
+ /** Call this to get the (merged) FieldInfos for a
+ * composite reader */
+ public static FieldInfos getMergedFieldInfos(IndexReader reader) {
+ final List<IndexReader> subReaders = new ArrayList<IndexReader>();
+ ReaderUtil.gatherSubReaders(subReaders, reader);
+ final FieldInfos fieldInfos = new FieldInfos();
+ for(IndexReader subReader : subReaders) {
+ fieldInfos.add(subReader.getFieldInfos());
+ }
+ return fieldInfos;
+ }
}