You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/10 16:05:21 UTC
svn commit: r1454862 [9/49] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/demo/
dev-tools/idea/lucene/facet/ dev-tools/idea/lucene/highlighter/ dev-too...
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentCoreReaders.java Sun Mar 10 15:04:57 2013
@@ -19,16 +19,19 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedHashSet;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.SegmentReader.CoreClosedListener;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
@@ -51,8 +54,8 @@ final class SegmentCoreReaders {
final FieldInfos fieldInfos;
final FieldsProducer fields;
- final PerDocProducer perDocProducer;
- final PerDocProducer norms;
+ final DocValuesProducer dvProducer;
+ final DocValuesProducer normsProducer;
final int termsIndexDivisor;
@@ -62,6 +65,10 @@ final class SegmentCoreReaders {
final TermVectorsReader termVectorsReaderOrig;
final CompoundFileDirectory cfsReader;
+ // TODO: make a single thread local w/ a
+ // Thingy class holding fieldsReader, termVectorsReader,
+ // normsProducer, dvProducer
+
final CloseableThreadLocal<StoredFieldsReader> fieldsReaderLocal = new CloseableThreadLocal<StoredFieldsReader>() {
@Override
protected StoredFieldsReader initialValue() {
@@ -72,11 +79,24 @@ final class SegmentCoreReaders {
final CloseableThreadLocal<TermVectorsReader> termVectorsLocal = new CloseableThreadLocal<TermVectorsReader>() {
@Override
protected TermVectorsReader initialValue() {
- return (termVectorsReaderOrig == null) ?
- null : termVectorsReaderOrig.clone();
+ return (termVectorsReaderOrig == null) ? null : termVectorsReaderOrig.clone();
}
};
-
+
+ final CloseableThreadLocal<Map<String,Object>> docValuesLocal = new CloseableThreadLocal<Map<String,Object>>() {
+ @Override
+ protected Map<String,Object> initialValue() {
+ return new HashMap<String,Object>();
+ }
+ };
+
+ final CloseableThreadLocal<Map<String,Object>> normsLocal = new CloseableThreadLocal<Map<String,Object>>() {
+ @Override
+ protected Map<String,Object> initialValue() {
+ return new HashMap<String,Object>();
+ }
+ };
+
private final Set<CoreClosedListener> coreClosedListeners =
Collections.synchronizedSet(new LinkedHashSet<CoreClosedListener>());
@@ -121,8 +141,20 @@ final class SegmentCoreReaders {
// ask codec for its Norms:
// TODO: since we don't write any norms file if there are no norms,
// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!
- norms = codec.normsFormat().docsProducer(segmentReadState);
- perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState);
+
+ if (fieldInfos.hasDocValues()) {
+ dvProducer = codec.docValuesFormat().fieldsProducer(segmentReadState);
+ assert dvProducer != null;
+ } else {
+ dvProducer = null;
+ }
+
+ if (fieldInfos.hasNorms()) {
+ normsProducer = codec.normsFormat().normsProducer(segmentReadState);
+ assert normsProducer != null;
+ } else {
+ normsProducer = null;
+ }
fieldsReaderOrig = info.getCodec().storedFieldsFormat().fieldsReader(cfsDir, info, fieldInfos, context);
@@ -149,17 +181,151 @@ final class SegmentCoreReaders {
void incRef() {
ref.incrementAndGet();
}
+
+ NumericDocValues getNumericDocValues(String field) throws IOException {
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null) {
+ // Field does not exist
+ return null;
+ }
+ if (fi.getDocValuesType() == null) {
+ // Field was not indexed with doc values
+ return null;
+ }
+ if (fi.getDocValuesType() != DocValuesType.NUMERIC) {
+ // DocValues were not numeric
+ return null;
+ }
+
+ assert dvProducer != null;
+
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ NumericDocValues dvs = (NumericDocValues) dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getNumeric(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
+ }
+
+ BinaryDocValues getBinaryDocValues(String field) throws IOException {
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null) {
+ // Field does not exist
+ return null;
+ }
+ if (fi.getDocValuesType() == null) {
+ // Field was not indexed with doc values
+ return null;
+ }
+ if (fi.getDocValuesType() != DocValuesType.BINARY) {
+ // DocValues were not binary
+ return null;
+ }
+
+ assert dvProducer != null;
+
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ BinaryDocValues dvs = (BinaryDocValues) dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getBinary(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
+ }
+
+ SortedDocValues getSortedDocValues(String field) throws IOException {
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null) {
+ // Field does not exist
+ return null;
+ }
+ if (fi.getDocValuesType() == null) {
+ // Field was not indexed with doc values
+ return null;
+ }
+ if (fi.getDocValuesType() != DocValuesType.SORTED) {
+ // DocValues were not sorted
+ return null;
+ }
+
+ assert dvProducer != null;
+
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ SortedDocValues dvs = (SortedDocValues) dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getSorted(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
+ }
+ SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null) {
+ // Field does not exist
+ return null;
+ }
+ if (fi.getDocValuesType() == null) {
+ // Field was not indexed with doc values
+ return null;
+ }
+ if (fi.getDocValuesType() != DocValuesType.SORTED_SET) {
+ // DocValues were not sorted
+ return null;
+ }
+
+ assert dvProducer != null;
+
+ Map<String,Object> dvFields = docValuesLocal.get();
+
+ SortedSetDocValues dvs = (SortedSetDocValues) dvFields.get(field);
+ if (dvs == null) {
+ dvs = dvProducer.getSortedSet(fi);
+ dvFields.put(field, dvs);
+ }
+
+ return dvs;
+ }
+
+ NumericDocValues getNormValues(String field) throws IOException {
+ FieldInfo fi = fieldInfos.fieldInfo(field);
+ if (fi == null) {
+ // Field does not exist
+ return null;
+ }
+ if (!fi.hasNorms()) {
+ return null;
+ }
+
+ assert normsProducer != null;
+
+ Map<String,Object> normFields = normsLocal.get();
+
+ NumericDocValues norms = (NumericDocValues) normFields.get(field);
+ if (norms == null) {
+ norms = normsProducer.getNumeric(fi);
+ normFields.put(field, norms);
+ }
+
+ return norms;
+ }
+
void decRef() throws IOException {
- //System.out.println("core.decRef seg=" + owner.getSegmentInfo() + " rc=" + ref);
if (ref.decrementAndGet() == 0) {
- IOUtils.close(termVectorsLocal, fieldsReaderLocal, fields, perDocProducer,
- termVectorsReaderOrig, fieldsReaderOrig, cfsReader, norms);
+ IOUtils.close(termVectorsLocal, fieldsReaderLocal, docValuesLocal, normsLocal, fields, dvProducer,
+ termVectorsReaderOrig, fieldsReaderOrig, cfsReader, normsProducer);
notifyCoreClosedListeners();
}
}
- private final void notifyCoreClosedListeners() {
+ private void notifyCoreClosedListeners() {
synchronized(coreClosedListeners) {
for (CoreClosedListener listener : coreClosedListeners) {
listener.onClose(owner);
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Sun Mar 10 15:04:57 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
-import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -57,8 +56,6 @@ public final class SegmentInfo {
private boolean isCompoundFile;
- private volatile long sizeInBytes = -1; // total byte size of all files (computed on demand)
-
private Codec codec;
private Map<String,String> diagnostics;
@@ -112,23 +109,6 @@ public final class SegmentInfo {
}
/**
- * Returns total size in bytes of all of files used by
- * this segment. Note that this will not include any live
- * docs for the segment; to include that use {@link
- * SegmentInfoPerCommit#sizeInBytes()} instead.
- */
- public long sizeInBytes() throws IOException {
- if (sizeInBytes == -1) {
- long sum = 0;
- for (final String fileName : files()) {
- sum += dir.fileLength(fileName);
- }
- sizeInBytes = sum;
- }
- return sizeInBytes;
- }
-
- /**
* Mark whether this segment is stored as a compound file.
*
* @param isCompoundFile true if this is a compound file;
@@ -265,7 +245,6 @@ public final class SegmentInfo {
public void setFiles(Set<String> files) {
checkFileNames(files);
setFiles = files;
- sizeInBytes = -1;
}
/** Add these files to the set of files written for this
@@ -273,7 +252,6 @@ public final class SegmentInfo {
public void addFiles(Collection<String> files) {
checkFileNames(files);
setFiles.addAll(files);
- sizeInBytes = -1;
}
/** Add this file to the set of files written for this
@@ -281,7 +259,6 @@ public final class SegmentInfo {
public void addFile(String file) {
checkFileNames(Collections.singleton(file));
setFiles.add(file);
- sizeInBytes = -1;
}
/** Remove this file from the set of files written for this
@@ -289,7 +266,6 @@ public final class SegmentInfo {
public void removeFile(String file) {
checkFileNames(Collections.singleton(file));
setFiles.remove(file);
- sizeInBytes = -1;
}
private void checkFileNames(Collection<String> files) {
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java Sun Mar 10 15:04:57 2013
@@ -43,6 +43,10 @@ public class SegmentInfoPerCommit {
// Generation number of updates (-1 if there are no updates yet):
private long updateGen;
+ // Normally 1+delGen, unless an exception was hit on last
+ // attempt to write:
+ private long nextWriteDelGen;
+
private volatile long sizeInBytes = -1;
/** Sole constructor.
@@ -56,14 +60,17 @@ public class SegmentInfoPerCommit {
this.delCount = delCount;
this.delGen = delGen;
this.updateGen = updateGen;
- }
-
- void advanceDelGen() {
if (delGen == -1) {
- delGen = 1;
+ nextWriteDelGen = 1;
} else {
- delGen++;
+ nextWriteDelGen = delGen+1;
}
+ }
+
+ /** Called when we succeed in writing deletes */
+ void advanceDelGen() {
+ delGen = nextWriteDelGen;
+ nextWriteDelGen = delGen+1;
sizeInBytes = -1;
}
@@ -76,6 +83,13 @@ public class SegmentInfoPerCommit {
sizeInBytes = -1;
}
+ /** Called if there was an exception while writing
+ * deletes, so that we don't try to write to the same
+ * file more than once. */
+ void advanceNextWriteDelGen() {
+ nextWriteDelGen++;
+ }
+
/** Returns total size in bytes of all files for this
* segment. */
public long sizeInBytes() throws IOException {
@@ -143,11 +157,7 @@ public class SegmentInfoPerCommit {
* of the live docs file.
*/
public long getNextDelGen() {
- if (delGen == -1) {
- return 1;
- } else {
- return delGen + 1;
- }
+ return nextWriteDelGen;
}
/**
@@ -189,7 +199,13 @@ public class SegmentInfoPerCommit {
@Override
public SegmentInfoPerCommit clone() {
- return new SegmentInfoPerCommit(info, delCount, delGen, updateGen);
+ SegmentInfoPerCommit other = new SegmentInfoPerCommit(info, delCount, delGen, updateGen);
+ // Not clear that we need to carry over nextWriteDelGen
+ // (i.e. do we ever clone after a failed write and
+ // before the next successful write?), but just do it to
+ // be safe:
+ other.nextWriteDelGen = nextWriteDelGen;
+ return other;
}
public void setUpdateGen(long updateGen) {
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Sun Mar 10 15:04:57 2013
@@ -870,9 +870,6 @@ public final class SegmentInfos implemen
// Ignore; this file is only used in a retry
// fallback on init.
}
- if (t instanceof ThreadInterruptedException) {
- throw (ThreadInterruptedException) t;
- }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Sun Mar 10 15:04:57 2013
@@ -19,16 +19,15 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.PerDocConsumer;
+import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.TermVectorsWriter;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
@@ -70,14 +69,14 @@ final class SegmentMerger {
/**
* Add an IndexReader to the collection of readers that are to be merged
*/
- final void add(IndexReader reader) {
+ void add(IndexReader reader) {
for (final AtomicReaderContext ctx : reader.leaves()) {
final AtomicReader r = ctx.reader();
mergeState.readers.add(r);
}
}
- final void add(SegmentReader reader) {
+ void add(SegmentReader reader) {
mergeState.readers.add(reader);
}
@@ -87,7 +86,7 @@ final class SegmentMerger {
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- final MergeState merge() throws IOException {
+ MergeState merge() throws IOException {
// NOTE: it's important to add calls to
// checkAbort.work(...) if you make any changes to this
// method that will spend alot of time. The frequency
@@ -96,22 +95,61 @@ final class SegmentMerger {
// threads.
mergeState.segmentInfo.setDocCount(setDocMaps());
- mergeDocValuesAndNormsFieldInfos();
+ mergeFieldInfos();
setMatchingSegmentReaders();
+ long t0 = 0;
+ if (mergeState.infoStream.isEnabled("SM")) {
+ t0 = System.nanoTime();
+ }
int numMerged = mergeFields();
+ if (mergeState.infoStream.isEnabled("SM")) {
+ long t1 = System.nanoTime();
+ mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge stored fields [" + numMerged + " docs]");
+ }
assert numMerged == mergeState.segmentInfo.getDocCount();
final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, mergeState.segmentInfo,
0, mergeState.fieldInfos, termIndexInterval, null, null, context);
+ if (mergeState.infoStream.isEnabled("SM")) {
+ t0 = System.nanoTime();
+ }
mergeTerms(segmentWriteState);
- mergePerDoc(segmentWriteState);
+ if (mergeState.infoStream.isEnabled("SM")) {
+ long t1 = System.nanoTime();
+ mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge postings [" + numMerged + " docs]");
+ }
+
+ if (mergeState.infoStream.isEnabled("SM")) {
+ t0 = System.nanoTime();
+ }
+ if (mergeState.fieldInfos.hasDocValues()) {
+ mergeDocValues(segmentWriteState);
+ }
+ if (mergeState.infoStream.isEnabled("SM")) {
+ long t1 = System.nanoTime();
+ mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge doc values [" + numMerged + " docs]");
+ }
if (mergeState.fieldInfos.hasNorms()) {
+ if (mergeState.infoStream.isEnabled("SM")) {
+ t0 = System.nanoTime();
+ }
mergeNorms(segmentWriteState);
+ if (mergeState.infoStream.isEnabled("SM")) {
+ long t1 = System.nanoTime();
+ mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge norms [" + numMerged + " docs]");
+ }
}
if (mergeState.fieldInfos.hasVectors()) {
+ if (mergeState.infoStream.isEnabled("SM")) {
+ t0 = System.nanoTime();
+ }
numMerged = mergeVectors();
+ if (mergeState.infoStream.isEnabled("SM")) {
+ long t1 = System.nanoTime();
+ mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge vectors [" + numMerged + " docs]");
+ }
assert numMerged == mergeState.segmentInfo.getDocCount();
}
@@ -122,6 +160,95 @@ final class SegmentMerger {
return mergeState;
}
+ private void mergeDocValues(SegmentWriteState segmentWriteState) throws IOException {
+ DocValuesConsumer consumer = codec.docValuesFormat().fieldsConsumer(segmentWriteState);
+ boolean success = false;
+ try {
+ for (FieldInfo field : mergeState.fieldInfos) {
+ DocValuesType type = field.getDocValuesType();
+ if (type != null) {
+ if (type == DocValuesType.NUMERIC) {
+ List<NumericDocValues> toMerge = new ArrayList<NumericDocValues>();
+ for (AtomicReader reader : mergeState.readers) {
+ NumericDocValues values = reader.getNumericDocValues(field.name);
+ if (values == null) {
+ values = NumericDocValues.EMPTY;
+ }
+ toMerge.add(values);
+ }
+ consumer.mergeNumericField(field, mergeState, toMerge);
+ } else if (type == DocValuesType.BINARY) {
+ List<BinaryDocValues> toMerge = new ArrayList<BinaryDocValues>();
+ for (AtomicReader reader : mergeState.readers) {
+ BinaryDocValues values = reader.getBinaryDocValues(field.name);
+ if (values == null) {
+ values = BinaryDocValues.EMPTY;
+ }
+ toMerge.add(values);
+ }
+ consumer.mergeBinaryField(field, mergeState, toMerge);
+ } else if (type == DocValuesType.SORTED) {
+ List<SortedDocValues> toMerge = new ArrayList<SortedDocValues>();
+ for (AtomicReader reader : mergeState.readers) {
+ SortedDocValues values = reader.getSortedDocValues(field.name);
+ if (values == null) {
+ values = SortedDocValues.EMPTY;
+ }
+ toMerge.add(values);
+ }
+ consumer.mergeSortedField(field, mergeState, toMerge);
+ } else if (type == DocValuesType.SORTED_SET) {
+ List<SortedSetDocValues> toMerge = new ArrayList<SortedSetDocValues>();
+ for (AtomicReader reader : mergeState.readers) {
+ SortedSetDocValues values = reader.getSortedSetDocValues(field.name);
+ if (values == null) {
+ values = SortedSetDocValues.EMPTY;
+ }
+ toMerge.add(values);
+ }
+ consumer.mergeSortedSetField(field, mergeState, toMerge);
+ } else {
+ throw new AssertionError("type=" + type);
+ }
+ }
+ }
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(consumer);
+ } else {
+ IOUtils.closeWhileHandlingException(consumer);
+ }
+ }
+ }
+
+ private void mergeNorms(SegmentWriteState segmentWriteState) throws IOException {
+ DocValuesConsumer consumer = codec.normsFormat().normsConsumer(segmentWriteState);
+ boolean success = false;
+ try {
+ for (FieldInfo field : mergeState.fieldInfos) {
+ if (field.hasNorms()) {
+ List<NumericDocValues> toMerge = new ArrayList<NumericDocValues>();
+ for (AtomicReader reader : mergeState.readers) {
+ NumericDocValues norms = reader.getNormValues(field.name);
+ if (norms == null) {
+ norms = NumericDocValues.EMPTY;
+ }
+ toMerge.add(norms);
+ }
+ consumer.mergeNumericField(field, mergeState, toMerge);
+ }
+ }
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(consumer);
+ } else {
+ IOUtils.closeWhileHandlingException(consumer);
+ }
+ }
+ }
+
private void setMatchingSegmentReaders() {
// If the i'th reader is a SegmentReader and has
// identical fieldName -> number mapping, then this
@@ -166,72 +293,15 @@ final class SegmentMerger {
}
}
- // returns an updated typepromoter (tracking type and size) given a previous one,
- // and a newly encountered docvalues
- private TypePromoter mergeDocValuesType(TypePromoter previous, DocValues docValues) {
- TypePromoter incoming = TypePromoter.create(docValues.getType(), docValues.getValueSize());
- if (previous == null) {
- previous = TypePromoter.getIdentityPromoter();
- }
- return previous.promote(incoming);
- }
-
- // NOTE: this is actually merging all the fieldinfos
- public void mergeDocValuesAndNormsFieldInfos() throws IOException {
- // mapping from all docvalues fields found to their promoted types
- // this is because FieldInfos does not store the
- // valueSize
- Map<FieldInfo,TypePromoter> docValuesTypes = new HashMap<FieldInfo,TypePromoter>();
- Map<FieldInfo,TypePromoter> normValuesTypes = new HashMap<FieldInfo,TypePromoter>();
-
+ public void mergeFieldInfos() throws IOException {
for (AtomicReader reader : mergeState.readers) {
FieldInfos readerFieldInfos = reader.getFieldInfos();
for (FieldInfo fi : readerFieldInfos) {
- FieldInfo merged = fieldInfosBuilder.add(fi);
- // update the type promotion mapping for this reader
- if (fi.hasDocValues()) {
- TypePromoter previous = docValuesTypes.get(merged);
- docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
- }
- if (fi.hasNorms()) {
- TypePromoter previous = normValuesTypes.get(merged);
- normValuesTypes.put(merged, mergeDocValuesType(previous, reader.normValues(fi.name)));
- }
+ fieldInfosBuilder.add(fi);
}
}
- updatePromoted(normValuesTypes, true);
- updatePromoted(docValuesTypes, false);
mergeState.fieldInfos = fieldInfosBuilder.finish();
}
-
- protected void updatePromoted(Map<FieldInfo,TypePromoter> infoAndPromoter, boolean norms) {
- // update any promoted doc values types:
- for (Map.Entry<FieldInfo,TypePromoter> e : infoAndPromoter.entrySet()) {
- FieldInfo fi = e.getKey();
- TypePromoter promoter = e.getValue();
- if (promoter == null) {
- if (norms) {
- fi.setNormValueType(null);
- } else {
- fi.setDocValuesType(null);
- }
- } else {
- assert promoter != TypePromoter.getIdentityPromoter();
- if (norms) {
- if (fi.getNormType() != promoter.type() && !fi.omitsNorms()) {
- // reset the type if we got promoted
- fi.setNormValueType(promoter.type());
- }
- } else {
- if (fi.getDocValuesType() != promoter.type()) {
- // reset the type if we got promoted
- fi.setDocValuesType(promoter.type());
- }
- }
- }
- }
- }
-
/**
*
@@ -253,7 +323,7 @@ final class SegmentMerger {
* Merge the TermVectors from each of the segments into the new one.
* @throws IOException if there is a low-level IO error
*/
- private final int mergeVectors() throws IOException {
+ private int mergeVectors() throws IOException {
final TermVectorsWriter termVectorsWriter = codec.termVectorsFormat().vectorsWriter(directory, mergeState.segmentInfo, context);
try {
@@ -289,7 +359,7 @@ final class SegmentMerger {
return docBase;
}
- private final void mergeTerms(SegmentWriteState segmentWriteState) throws IOException {
+ private void mergeTerms(SegmentWriteState segmentWriteState) throws IOException {
final List<Fields> fields = new ArrayList<Fields>();
final List<ReaderSlice> slices = new ArrayList<ReaderSlice>();
@@ -322,38 +392,4 @@ final class SegmentMerger {
}
}
}
-
- private void mergePerDoc(SegmentWriteState segmentWriteState) throws IOException {
- final PerDocConsumer docsConsumer = codec.docValuesFormat()
- .docsConsumer(new PerDocWriteState(segmentWriteState));
- assert docsConsumer != null;
- boolean success = false;
- try {
- docsConsumer.merge(mergeState);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(docsConsumer);
- } else {
- IOUtils.closeWhileHandlingException(docsConsumer);
- }
- }
- }
-
- private void mergeNorms(SegmentWriteState segmentWriteState) throws IOException {
- final PerDocConsumer docsConsumer = codec.normsFormat()
- .docsConsumer(new PerDocWriteState(segmentWriteState));
- assert docsConsumer != null;
- boolean success = false;
- try {
- docsConsumer.merge(mergeState);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(docsConsumer);
- } else {
- IOUtils.closeWhileHandlingException(docsConsumer);
- }
- }
- }
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReadState.java Sun Mar 10 15:04:57 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.store.IOContext
*/
public class SegmentReadState {
/** {@link Directory} where this segment is read from. */
- public final Directory dir;
+ public final Directory directory;
/** {@link SegmentInfo} describing this segment. */
public final SegmentInfo segmentInfo;
@@ -73,7 +73,7 @@ public class SegmentReadState {
IOContext context,
int termsIndexDivisor,
String segmentSuffix) {
- this.dir = dir;
+ this.directory = dir;
this.segmentInfo = info;
this.fieldInfos = fieldInfos;
this.context = context;
@@ -84,7 +84,7 @@ public class SegmentReadState {
/** Create a {@code SegmentReadState}. */
public SegmentReadState(SegmentReadState other,
String newSegmentSuffix) {
- this.dir = other.dir;
+ this.directory = other.directory;
this.segmentInfo = other.segmentInfo;
this.fieldInfos = other.fieldInfos;
this.context = other.context;
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Sun Mar 10 15:04:57 2013
@@ -22,7 +22,6 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.search.FieldCache;
@@ -427,46 +426,36 @@ public final class SegmentReader extends
public int getTermInfosIndexDivisor() {
return core.termsIndexDivisor;
}
-
+
@Override
- public DocValues docValues(String field) throws IOException {
+ public NumericDocValues getNumericDocValues(String field) throws IOException {
+ return core.getNumericDocValues(field);
+ }
+
+ @Override
+ public BinaryDocValues getBinaryDocValues(String field) throws IOException {
ensureOpen();
- DocValues docValues = internalGetDocValues(core.perDocProducer, field);
- if (updates != null) {
- // if no norms for core, try using the first producer available in updates
- for (int i = 0; docValues == null && i < updates.length; i++) {
- if (updates[i] != null) {
- docValues = internalGetDocValues(updates[i].perDocProducer, field);
- }
- }
- }
- return docValues;
+ return core.getBinaryDocValues(field);
}
@Override
- public DocValues normValues(String field) throws IOException {
+ public SortedDocValues getSortedDocValues(String field) throws IOException {
ensureOpen();
- DocValues normValues = internalGetDocValues(core.norms, field);
- if (updates != null) {
- // if no norms for core, try using the first norms available in updates
- for (int i = 0; normValues == null && i < updates.length; i++) {
- if (updates[i] != null) {
- normValues = internalGetDocValues(updates[i].norms, field);
- }
- }
- }
- return normValues;
+ return core.getSortedDocValues(field);
}
-
- private DocValues internalGetDocValues(PerDocProducer perDoc, String field)
- throws IOException {
- if (perDoc == null) {
- return null;
- }
- final DocValues docValues = perDoc.docValues(field);
- return docValues;
+
+ @Override
+ public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
+ ensureOpen();
+ return core.getSortedSetDocValues(field);
}
-
+
+ @Override
+ public NumericDocValues getNormValues(String field) throws IOException {
+ ensureOpen();
+ return core.getNormValues(field);
+ }
+
/**
* Called when the shared core for this SegmentReader is closed.
* <p>
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Sun Mar 10 15:04:57 2013
@@ -24,6 +24,10 @@ import java.util.Map;
import org.apache.lucene.util.Bits;
import org.apache.lucene.index.DirectoryReader; // javadoc
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
+import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
+import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.MultiReader; // javadoc
/**
@@ -44,7 +48,6 @@ import org.apache.lucene.index.MultiRead
public final class SlowCompositeReaderWrapper extends AtomicReader {
private final CompositeReader in;
- private final Map<String, DocValues> normsCache = new HashMap<String, DocValues>();
private final Fields fields;
private final Bits liveDocs;
@@ -83,25 +86,106 @@ public final class SlowCompositeReaderWr
}
@Override
- public DocValues docValues(String field) throws IOException {
+ public NumericDocValues getNumericDocValues(String field) throws IOException {
ensureOpen();
- return MultiDocValues.getDocValues(in, field);
+ return MultiDocValues.getNumericValues(in, field);
+ }
+
+ @Override
+ public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+ ensureOpen();
+ return MultiDocValues.getBinaryValues(in, field);
+ }
+
+ @Override
+ public SortedDocValues getSortedDocValues(String field) throws IOException {
+ ensureOpen();
+ OrdinalMap map = null;
+ synchronized (cachedOrdMaps) {
+ map = cachedOrdMaps.get(field);
+ if (map == null) {
+ // uncached, or not a multi dv
+ SortedDocValues dv = MultiDocValues.getSortedValues(in, field);
+ if (dv instanceof MultiSortedDocValues) {
+ map = ((MultiSortedDocValues)dv).mapping;
+ if (map.owner == getCoreCacheKey()) {
+ cachedOrdMaps.put(field, map);
+ }
+ }
+ return dv;
+ }
+ }
+ // cached ordinal map
+ if (getFieldInfos().fieldInfo(field).getDocValuesType() != DocValuesType.SORTED) {
+ return null;
+ }
+ int size = in.leaves().size();
+ final SortedDocValues[] values = new SortedDocValues[size];
+ final int[] starts = new int[size+1];
+ for (int i = 0; i < size; i++) {
+ AtomicReaderContext context = in.leaves().get(i);
+ SortedDocValues v = context.reader().getSortedDocValues(field);
+ if (v == null) {
+ v = SortedDocValues.EMPTY;
+ }
+ values[i] = v;
+ starts[i] = context.docBase;
+ }
+ starts[size] = maxDoc();
+ return new MultiSortedDocValues(values, starts, map);
}
@Override
- public synchronized DocValues normValues(String field) throws IOException {
+ public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
ensureOpen();
- DocValues values = normsCache.get(field);
- if (values == null) {
- values = MultiDocValues.getNormDocValues(in, field);
- normsCache.put(field, values);
+ OrdinalMap map = null;
+ synchronized (cachedOrdMaps) {
+ map = cachedOrdMaps.get(field);
+ if (map == null) {
+ // uncached, or not a multi dv
+ SortedSetDocValues dv = MultiDocValues.getSortedSetValues(in, field);
+ if (dv instanceof MultiSortedSetDocValues) {
+ map = ((MultiSortedSetDocValues)dv).mapping;
+ if (map.owner == getCoreCacheKey()) {
+ cachedOrdMaps.put(field, map);
+ }
+ }
+ return dv;
+ }
+ }
+ // cached ordinal map
+ if (getFieldInfos().fieldInfo(field).getDocValuesType() != DocValuesType.SORTED_SET) {
+ return null;
}
- return values;
+ assert map != null;
+ int size = in.leaves().size();
+ final SortedSetDocValues[] values = new SortedSetDocValues[size];
+ final int[] starts = new int[size+1];
+ for (int i = 0; i < size; i++) {
+ AtomicReaderContext context = in.leaves().get(i);
+ SortedSetDocValues v = context.reader().getSortedSetDocValues(field);
+ if (v == null) {
+ v = SortedSetDocValues.EMPTY;
+ }
+ values[i] = v;
+ starts[i] = context.docBase;
+ }
+ starts[size] = maxDoc();
+ return new MultiSortedSetDocValues(values, starts, map);
+ }
+
+ // TODO: this could really be a weak map somewhere else on the coreCacheKey,
+ // but do we really need to optimize slow-wrapper any more?
+ private final Map<String,OrdinalMap> cachedOrdMaps = new HashMap<String,OrdinalMap>();
+
+ @Override
+ public NumericDocValues getNormValues(String field) throws IOException {
+ ensureOpen();
+ return MultiDocValues.getNormValues(in, field);
}
@Override
- public Fields getTermVectors(int docID)
- throws IOException {
+ public Fields getTermVectors(int docID) throws IOException {
ensureOpen();
return in.getTermVectors(docID);
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StackedTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StackedTerms.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StackedTerms.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StackedTerms.java Sun Mar 10 15:04:57 2013
@@ -62,7 +62,7 @@ public class StackedTerms extends Terms
}
@Override
- public Comparator<BytesRef> getComparator() throws IOException {
+ public Comparator<BytesRef> getComparator() {
if (comparator == null) {
for (int i = 0; i < subTerms.length; i++) {
if (subTerms[i] != null) {
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java Sun Mar 10 15:04:57 2013
@@ -19,129 +19,10 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.RamUsageEstimator;
-
-/** This is a DocFieldConsumer that writes stored fields. */
-final class StoredFieldsConsumer {
-
- StoredFieldsWriter fieldsWriter;
- final DocumentsWriterPerThread docWriter;
- int lastDocID;
-
- int freeCount;
-
- final DocumentsWriterPerThread.DocState docState;
- final Codec codec;
-
- public StoredFieldsConsumer(DocumentsWriterPerThread docWriter) {
- this.docWriter = docWriter;
- this.docState = docWriter.docState;
- this.codec = docWriter.codec;
- }
-
- private int numStoredFields;
- private StorableField[] storedFields;
- private FieldInfo[] fieldInfos;
-
- public void reset() {
- numStoredFields = 0;
- storedFields = new StorableField[1];
- fieldInfos = new FieldInfo[1];
- }
-
- public void startDocument() {
- reset();
- }
-
- public void flush(SegmentWriteState state) throws IOException {
- int numDocs = state.segmentInfo.getDocCount();
-
- if (numDocs > 0) {
- // It's possible that all documents seen in this segment
- // hit non-aborting exceptions, in which case we will
- // not have yet init'd the FieldsWriter:
- initFieldsWriter(state.context);
- fill(numDocs);
- }
-
- if (fieldsWriter != null) {
- try {
- fieldsWriter.finish(state.fieldInfos, numDocs);
- } finally {
- fieldsWriter.close();
- fieldsWriter = null;
- lastDocID = 0;
- }
- }
- }
-
- private synchronized void initFieldsWriter(IOContext context) throws IOException {
- if (fieldsWriter == null) {
- fieldsWriter = codec.storedFieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
- lastDocID = 0;
- }
- }
-
- int allocCount;
-
- void abort() {
- reset();
-
- if (fieldsWriter != null) {
- fieldsWriter.abort();
- fieldsWriter = null;
- lastDocID = 0;
- }
- }
-
- /** Fills in any hole in the docIDs */
- void fill(int docID) throws IOException {
- // We must "catch up" for all docs before us
- // that had no stored fields:
- while(lastDocID < docID) {
- fieldsWriter.startDocument(0);
- lastDocID++;
- }
- }
-
- void finishDocument() throws IOException {
- assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument start");
-
- initFieldsWriter(IOContext.DEFAULT);
- fill(docState.docID);
-
- if (fieldsWriter != null && numStoredFields > 0) {
- fieldsWriter.startDocument(numStoredFields);
- for (int i = 0; i < numStoredFields; i++) {
- fieldsWriter.writeField(fieldInfos[i], storedFields[i]);
- }
- lastDocID++;
- }
-
- reset();
- assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end");
- }
-
- public void addField(StorableField field, FieldInfo fieldInfo) {
- if (numStoredFields == storedFields.length) {
- int newSize = ArrayUtil.oversize(numStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- StorableField[] newArray = new StorableField[newSize];
- System.arraycopy(storedFields, 0, newArray, 0, numStoredFields);
- storedFields = newArray;
-
- FieldInfo[] newInfoArray = new FieldInfo[newSize];
- System.arraycopy(fieldInfos, 0, newInfoArray, 0, numStoredFields);
- fieldInfos = newInfoArray;
- }
-
- storedFields[numStoredFields] = field;
- fieldInfos[numStoredFields] = fieldInfo;
- numStoredFields++;
-
- assert docState.testPoint("StoredFieldsWriterPerThread.processFields.writeField");
- }
+abstract class StoredFieldsConsumer {
+ abstract void addField(int docID, StorableField field, FieldInfo fieldInfo) throws IOException;
+ abstract void flush(SegmentWriteState state) throws IOException;
+ abstract void abort() throws IOException;
+ abstract void startDocument() throws IOException;
+ abstract void finishDocument() throws IOException;
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java Sun Mar 10 15:04:57 2013
@@ -78,6 +78,7 @@ final class TermVectorsConsumer extends
void fill(int docID) throws IOException {
while(lastDocID < docID) {
writer.startDocument(0);
+ writer.finishDocument();
lastDocID++;
}
}
@@ -108,6 +109,7 @@ final class TermVectorsConsumer extends
for (int i = 0; i < numVectorFields; i++) {
perFields[i].finishDocument();
}
+ writer.finishDocument();
assert lastDocID == docState.docID: "lastDocID=" + lastDocID + " docState.docID=" + docState.docID;
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Sun Mar 10 15:04:57 2013
@@ -182,11 +182,12 @@ final class TermVectorsConsumerPerField
}
tv.addProx(freq, posReader, offReader);
}
+ tv.finishTerm();
}
+ tv.finishField();
termsHashPerField.reset();
- // commit the termVectors once successful - FI will otherwise reset them
fieldInfo.setStoreTermVectors();
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/Terms.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/Terms.java Sun Mar 10 15:04:57 2013
@@ -80,7 +80,7 @@ public abstract class Terms {
* if there are no terms. This method may be invoked
* many times; it's best to cache a single instance &
* reuse it. */
- public abstract Comparator<BytesRef> getComparator() throws IOException;
+ public abstract Comparator<BytesRef> getComparator();
/** Returns the number of terms for this field, or -1 if this
* measure isn't stored by the codec. Note that, just like
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHash.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TermsHash.java Sun Mar 10 15:04:57 2013
@@ -126,11 +126,6 @@ final class TermsHash extends InvertedDo
}
@Override
- public boolean freeRAM() {
- return false;
- }
-
- @Override
void finishDocument() throws IOException {
consumer.finishDocument(this);
if (nextTermsHash != null) {
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java Sun Mar 10 15:04:57 2013
@@ -374,7 +374,7 @@ public class TieredMergePolicy extends M
for(int idx = tooBigCount; idx<infosSorted.size(); idx++) {
final SegmentInfoPerCommit info = infosSorted.get(idx);
if (merging.contains(info)) {
- mergingBytes += info.info.sizeInBytes();
+ mergingBytes += info.sizeInBytes();
} else if (!toBeMerged.contains(info)) {
eligible.add(info);
}
@@ -470,7 +470,7 @@ public class TieredMergePolicy extends M
final long segBytes = size(info);
totAfterMergeBytes += segBytes;
totAfterMergeBytesFloored += floorSize(segBytes);
- totBeforeMergeBytes += info.info.sizeInBytes();
+ totBeforeMergeBytes += info.sizeInBytes();
}
// Measure "skew" of the merge, which can range
@@ -670,7 +670,7 @@ public class TieredMergePolicy extends M
// Segment size in bytes, pro-rated by % deleted
private long size(SegmentInfoPerCommit info) throws IOException {
- final long byteSize = info.info.sizeInBytes();
+ final long byteSize = info.sizeInBytes();
final int delCount = writer.get().numDeletedDocs(info);
final double delRatio = (info.info.getDocCount() <= 0 ? 0.0f : ((double)delCount / (double)info.info.getDocCount()));
assert delRatio <= 1.0;
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/package.html?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/package.html (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/index/package.html Sun Mar 10 15:04:57 2013
@@ -254,7 +254,7 @@ its {@link org.apache.lucene.search.simi
</p>
<p>
Additional user-supplied statistics can be added to the document as DocValues fields and
-accessed via {@link org.apache.lucene.index.AtomicReader#docValues}.
+accessed via {@link org.apache.lucene.index.AtomicReader#getNumericDocValues}.
</p>
<p>
</body>
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Sun Mar 10 15:04:57 2013
@@ -50,26 +50,32 @@ class ConjunctionTermScorer extends Scor
}
private int doNext(int doc) throws IOException {
- do {
- if (lead.doc == DocIdSetIterator.NO_MORE_DOCS) {
- return NO_MORE_DOCS;
- }
- advanceHead: do {
+ for(;;) {
+ // doc may already be NO_MORE_DOCS here, but we don't check explicitly
+ // since all scorers should advance to NO_MORE_DOCS, match, then
+ // return that value.
+ advanceHead: for(;;) {
for (int i = 1; i < docsAndFreqs.length; i++) {
+ // invariant: docsAndFreqs[i].doc <= doc at this point.
+
+ // docsAndFreqs[i].doc may already be equal to doc if we "broke advanceHead"
+ // on the previous iteration and the advance on the lead scorer exactly matched.
if (docsAndFreqs[i].doc < doc) {
docsAndFreqs[i].doc = docsAndFreqs[i].docs.advance(doc);
- }
- if (docsAndFreqs[i].doc > doc) {
- // DocsEnum beyond the current doc - break and advance lead
- break advanceHead;
+
+ if (docsAndFreqs[i].doc > doc) {
+ // DocsEnum beyond the current doc - break and advance lead to the new highest doc.
+ doc = docsAndFreqs[i].doc;
+ break advanceHead;
+ }
}
}
// success - all DocsEnums are on the same doc
return doc;
- } while (true);
+ }
// advance head for next iteration
- doc = lead.doc = lead.docs.nextDoc();
- } while (true);
+ doc = lead.doc = lead.docs.advance(doc);
+ }
}
@Override
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java Sun Mar 10 15:04:57 2013
@@ -24,6 +24,8 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Set;
/**
@@ -248,6 +250,14 @@ public class ConstantScoreQuery extends
return super.score(collector, max, firstDocID);
}
}
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ if (docIdSetIterator instanceof Scorer)
+ return Collections.singletonList(new ChildScorer((Scorer) docIdSetIterator, "constant"));
+ else
+ return Collections.emptyList();
+ }
}
@Override
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/FieldCache.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/search/FieldCache.java Sun Mar 10 15:04:57 2013
@@ -19,21 +19,23 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.io.PrintStream;
-import java.text.DecimalFormat;
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
-import org.apache.lucene.document.IntField; // for javadocs
+import org.apache.lucene.document.DoubleField; // for javadocs
import org.apache.lucene.document.FloatField; // for javadocs
+import org.apache.lucene.document.IntField; // for javadocs
import org.apache.lucene.document.LongField; // for javadocs
-import org.apache.lucene.document.DoubleField; // for javadocs
-import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocTermOrds;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.packed.PackedInts;
/**
* Expert: Maintains caches of term values.
@@ -42,9 +44,115 @@ import org.apache.lucene.util.packed.Pac
*
* @since lucene 1.4
* @see org.apache.lucene.util.FieldCacheSanityChecker
+ *
+ * @lucene.internal
*/
public interface FieldCache {
+ /** Field values as 8-bit signed bytes */
+ public static abstract class Bytes {
+ /** Return a single Byte representation of this field's value. */
+ public abstract byte get(int docID);
+
+ /** Zero value for every document */
+ public static final Bytes EMPTY = new Bytes() {
+ @Override
+ public byte get(int docID) {
+ return 0;
+ }
+ };
+ }
+
+ /** Field values as 16-bit signed shorts */
+ public static abstract class Shorts {
+ /** Return a short representation of this field's value. */
+ public abstract short get(int docID);
+
+ /** Zero value for every document */
+ public static final Shorts EMPTY = new Shorts() {
+ @Override
+ public short get(int docID) {
+ return 0;
+ }
+ };
+ }
+
+ /** Field values as 32-bit signed integers */
+ public static abstract class Ints {
+ /** Return an integer representation of this field's value. */
+ public abstract int get(int docID);
+
+ /** Zero value for every document */
+ public static final Ints EMPTY = new Ints() {
+ @Override
+ public int get(int docID) {
+ return 0;
+ }
+ };
+ }
+
+ /** Field values as 64-bit signed long integers */
+ public static abstract class Longs {
+ /** Return an long representation of this field's value. */
+ public abstract long get(int docID);
+
+ /** Zero value for every document */
+ public static final Longs EMPTY = new Longs() {
+ @Override
+ public long get(int docID) {
+ return 0;
+ }
+ };
+ }
+
+ /** Field values as 32-bit floats */
+ public static abstract class Floats {
+ /** Return an float representation of this field's value. */
+ public abstract float get(int docID);
+
+ /** Zero value for every document */
+ public static final Floats EMPTY = new Floats() {
+ @Override
+ public float get(int docID) {
+ return 0;
+ }
+ };
+ }
+
+ /** Field values as 64-bit doubles */
+ public static abstract class Doubles {
+ /** Return an double representation of this field's value. */
+ public abstract double get(int docID);
+
+ /** Zero value for every document */
+ public static final Doubles EMPTY = new Doubles() {
+ @Override
+ public double get(int docID) {
+ return 0;
+ }
+ };
+ }
+
+ /** Returns MISSING/-1 ordinal for every document */
+ public static final SortedDocValues EMPTY_TERMSINDEX = new SortedDocValues() {
+ @Override
+ public int getOrd(int docID) {
+ return -1;
+ }
+
+ @Override
+ public void lookupOrd(int ord, BytesRef result) {
+ result.bytes = MISSING;
+ result.offset = 0;
+ result.length = 0;
+ }
+
+ @Override
+ public int getValueCount() {
+ return 0;
+ }
+ };
+
/**
* Placeholder indicating creation of this cache is currently in-progress.
*/
@@ -53,20 +161,21 @@ public interface FieldCache {
}
/**
- * Hack: When thrown from a Parser (NUMERIC_UTILS_* ones), this stops
- * processing terms and returns the current FieldCache
- * array.
- * @lucene.internal
- */
- public static final class StopFillCacheException extends RuntimeException {
- }
-
- /**
* Marker interface as super-interface to all parsers. It
* is used to specify a custom parser to {@link
* SortField#SortField(String, FieldCache.Parser)}.
*/
public interface Parser {
+
+ /**
+ * Pulls a {@link TermsEnum} from the given {@link Terms}. This method allows certain parsers
+ * to filter the actual TermsEnum before the field cache is filled.
+ *
+ * @param terms the {@link Terms} instance to create the {@link TermsEnum} from.
+ * @return a possibly filtered {@link TermsEnum} instance, this method must not return <code>null</code>.
+ * @throws IOException if an {@link IOException} occurs
+ */
+ public TermsEnum termsEnum(Terms terms) throws IOException;
}
/** Interface to parse bytes from document fields.
@@ -113,7 +222,7 @@ public interface FieldCache {
* @see FieldCache#getDoubles(AtomicReader, String, FieldCache.DoubleParser, boolean)
*/
public interface DoubleParser extends Parser {
- /** Return an long representation of this field's value. */
+ /** Return an double representation of this field's value. */
public double parseDouble(BytesRef term);
}
@@ -134,6 +243,10 @@ public interface FieldCache {
public String toString() {
return FieldCache.class.getName()+".DEFAULT_BYTE_PARSER";
}
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return terms.iterator(null);
+ }
};
/** The default parser for short values, which are encoded by {@link Short#toString(short)} */
@@ -150,6 +263,11 @@ public interface FieldCache {
public String toString() {
return FieldCache.class.getName()+".DEFAULT_SHORT_PARSER";
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return terms.iterator(null);
+ }
};
/** The default parser for int values, which are encoded by {@link Integer#toString(int)} */
@@ -162,6 +280,12 @@ public interface FieldCache {
// directly from byte[]
return Integer.parseInt(term.utf8ToString());
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return terms.iterator(null);
+ }
+
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_INT_PARSER";
@@ -178,6 +302,12 @@ public interface FieldCache {
// directly from byte[]
return Float.parseFloat(term.utf8ToString());
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return terms.iterator(null);
+ }
+
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_FLOAT_PARSER";
@@ -194,6 +324,12 @@ public interface FieldCache {
// directly from byte[]
return Long.parseLong(term.utf8ToString());
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return terms.iterator(null);
+ }
+
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_LONG_PARSER";
@@ -210,6 +346,12 @@ public interface FieldCache {
// directly from byte[]
return Double.parseDouble(term.utf8ToString());
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return terms.iterator(null);
+ }
+
@Override
public String toString() {
return FieldCache.class.getName()+".DEFAULT_DOUBLE_PARSER";
@@ -223,10 +365,14 @@ public interface FieldCache {
public static final IntParser NUMERIC_UTILS_INT_PARSER=new IntParser(){
@Override
public int parseInt(BytesRef term) {
- if (NumericUtils.getPrefixCodedIntShift(term) > 0)
- throw new StopFillCacheException();
return NumericUtils.prefixCodedToInt(term);
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return NumericUtils.filterPrefixCodedInts(terms.iterator(null));
+ }
+
@Override
public String toString() {
return FieldCache.class.getName()+".NUMERIC_UTILS_INT_PARSER";
@@ -240,14 +386,17 @@ public interface FieldCache {
public static final FloatParser NUMERIC_UTILS_FLOAT_PARSER=new FloatParser(){
@Override
public float parseFloat(BytesRef term) {
- if (NumericUtils.getPrefixCodedIntShift(term) > 0)
- throw new StopFillCacheException();
return NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term));
}
@Override
public String toString() {
return FieldCache.class.getName()+".NUMERIC_UTILS_FLOAT_PARSER";
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return NumericUtils.filterPrefixCodedInts(terms.iterator(null));
+ }
};
/**
@@ -257,14 +406,17 @@ public interface FieldCache {
public static final LongParser NUMERIC_UTILS_LONG_PARSER = new LongParser(){
@Override
public long parseLong(BytesRef term) {
- if (NumericUtils.getPrefixCodedLongShift(term) > 0)
- throw new StopFillCacheException();
return NumericUtils.prefixCodedToLong(term);
}
@Override
public String toString() {
return FieldCache.class.getName()+".NUMERIC_UTILS_LONG_PARSER";
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return NumericUtils.filterPrefixCodedLongs(terms.iterator(null));
+ }
};
/**
@@ -274,24 +426,28 @@ public interface FieldCache {
public static final DoubleParser NUMERIC_UTILS_DOUBLE_PARSER = new DoubleParser(){
@Override
public double parseDouble(BytesRef term) {
- if (NumericUtils.getPrefixCodedLongShift(term) > 0)
- throw new StopFillCacheException();
return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term));
}
@Override
public String toString() {
return FieldCache.class.getName()+".NUMERIC_UTILS_DOUBLE_PARSER";
}
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return NumericUtils.filterPrefixCodedLongs(terms.iterator(null));
+ }
};
/** Checks the internal cache for an appropriate entry, and if none is found,
- * reads the terms in <code>field</code> and returns a bit set at the size of
- * <code>reader.maxDoc()</code>, with turned on bits for each docid that
- * does have a value for this field.
+ * reads the terms in <code>field</code> and returns a bit set at the size of
+ * <code>reader.maxDoc()</code>, with turned on bits for each docid that
+ * does have a value for this field. Note that if the field was only indexed
+ * as DocValues then this method will not work (it will return a Bits stating
+ * that no documents contain the field).
*/
- public Bits getDocsWithField(AtomicReader reader, String field)
- throws IOException;
+ public Bits getDocsWithField(AtomicReader reader, String field) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as a single byte and returns an array
@@ -304,8 +460,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public byte[] getBytes (AtomicReader reader, String field, boolean setDocsWithField)
- throws IOException;
+ public Bytes getBytes(AtomicReader reader, String field, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as bytes and returns an array of
@@ -319,8 +474,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public byte[] getBytes (AtomicReader reader, String field, ByteParser parser, boolean setDocsWithField)
- throws IOException;
+ public Bytes getBytes(AtomicReader reader, String field, ByteParser parser, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as shorts and returns an array
@@ -333,8 +487,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public short[] getShorts (AtomicReader reader, String field, boolean setDocsWithField)
- throws IOException;
+ public Shorts getShorts (AtomicReader reader, String field, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as shorts and returns an array of
@@ -348,8 +501,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public short[] getShorts (AtomicReader reader, String field, ShortParser parser, boolean setDocsWithField)
- throws IOException;
+ public Shorts getShorts (AtomicReader reader, String field, ShortParser parser, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is
* found, reads the terms in <code>field</code> as integers and returns an array
@@ -362,8 +514,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public int[] getInts (AtomicReader reader, String field, boolean setDocsWithField)
- throws IOException;
+ public Ints getInts (AtomicReader reader, String field, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none is found,
* reads the terms in <code>field</code> as integers and returns an array of
@@ -377,8 +528,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public int[] getInts (AtomicReader reader, String field, IntParser parser, boolean setDocsWithField)
- throws IOException;
+ public Ints getInts (AtomicReader reader, String field, IntParser parser, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if
* none is found, reads the terms in <code>field</code> as floats and returns an array
@@ -391,8 +541,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public float[] getFloats (AtomicReader reader, String field, boolean setDocsWithField)
- throws IOException;
+ public Floats getFloats (AtomicReader reader, String field, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if
* none is found, reads the terms in <code>field</code> as floats and returns an array
@@ -406,8 +555,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public float[] getFloats (AtomicReader reader, String field,
- FloatParser parser, boolean setDocsWithField) throws IOException;
+ public Floats getFloats (AtomicReader reader, String field, FloatParser parser, boolean setDocsWithField) throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is
@@ -422,7 +570,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws java.io.IOException If any error occurs.
*/
- public long[] getLongs(AtomicReader reader, String field, boolean setDocsWithField)
+ public Longs getLongs(AtomicReader reader, String field, boolean setDocsWithField)
throws IOException;
/**
@@ -439,7 +587,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public long[] getLongs(AtomicReader reader, String field, LongParser parser, boolean setDocsWithField)
+ public Longs getLongs(AtomicReader reader, String field, LongParser parser, boolean setDocsWithField)
throws IOException;
/**
@@ -455,7 +603,7 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public double[] getDoubles(AtomicReader reader, String field, boolean setDocsWithField)
+ public Doubles getDoubles(AtomicReader reader, String field, boolean setDocsWithField)
throws IOException;
/**
@@ -472,35 +620,18 @@ public interface FieldCache {
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public double[] getDoubles(AtomicReader reader, String field, DoubleParser parser, boolean setDocsWithField)
- throws IOException;
-
- /** Returned by {@link #getTerms} */
- public abstract static class DocTerms {
- /** The BytesRef argument must not be null; the method
- * returns the same BytesRef, or an empty (length=0)
- * BytesRef if the doc did not have this field or was
- * deleted. */
- public abstract BytesRef getTerm(int docID, BytesRef ret);
-
- /** Returns true if this doc has this field and is not
- * deleted. */
- public abstract boolean exists(int docID);
-
- /** Number of documents */
- public abstract int size();
- }
+ public Doubles getDoubles(AtomicReader reader, String field, DoubleParser parser, boolean setDocsWithField) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none
* is found, reads the term values in <code>field</code>
- * and returns a {@link DocTerms} instance, providing a
+ * and returns a {@link BinaryDocValues} instance, providing a
* method to retrieve the term (as a BytesRef) per document.
* @param reader Used to get field values.
* @param field Which field contains the strings.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public DocTerms getTerms (AtomicReader reader, String field)
+ public BinaryDocValues getTerms (AtomicReader reader, String field)
throws IOException;
/** Expert: just like {@link #getTerms(AtomicReader,String)},
@@ -508,76 +639,19 @@ public interface FieldCache {
* faster lookups (default is "true"). Note that the
* first call for a given reader and field "wins",
* subsequent calls will share the same cache entry. */
- public DocTerms getTerms (AtomicReader reader, String field, float acceptableOverheadRatio)
- throws IOException;
-
- /** Returned by {@link #getTermsIndex} */
- public abstract static class DocTermsIndex {
-
- public int binarySearchLookup(BytesRef key, BytesRef spare) {
- // this special case is the reason that Arrays.binarySearch() isn't useful.
- if (key == null)
- return 0;
-
- int low = 1;
- int high = numOrd()-1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
- int cmp = lookup(mid, spare).compareTo(key);
-
- if (cmp < 0)
- low = mid + 1;
- else if (cmp > 0)
- high = mid - 1;
- else
- return mid; // key found
- }
- return -(low + 1); // key not found.
- }
-
- /** The BytesRef argument must not be null; the method
- * returns the same BytesRef, or an empty (length=0)
- * BytesRef if this ord is the null ord (0). */
- public abstract BytesRef lookup(int ord, BytesRef reuse);
-
- /** Convenience method, to lookup the Term for a doc.
- * If this doc is deleted or did not have this field,
- * this will return an empty (length=0) BytesRef. */
- public BytesRef getTerm(int docID, BytesRef reuse) {
- return lookup(getOrd(docID), reuse);
- }
-
- /** Returns sort ord for this document. Ord 0 is
- * reserved for docs that are deleted or did not have
- * this field. */
- public abstract int getOrd(int docID);
-
- /** Returns total unique ord count; this includes +1 for
- * the null ord (always 0). */
- public abstract int numOrd();
-
- /** Number of documents */
- public abstract int size();
-
- /** Returns a TermsEnum that can iterate over the values in this index entry */
- public abstract TermsEnum getTermsEnum();
-
- /** @lucene.internal */
- public abstract PackedInts.Reader getDocToOrd();
- }
+ public BinaryDocValues getTerms (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException;
/** Checks the internal cache for an appropriate entry, and if none
* is found, reads the term values in <code>field</code>
- * and returns a {@link DocTerms} instance, providing a
- * method to retrieve the term (as a BytesRef) per document.
+ * and returns a {@link SortedDocValues} instance,
+ * providing methods to retrieve sort ordinals and terms
+ * (as a ByteRef) per document.
* @param reader Used to get field values.
* @param field Which field contains the strings.
* @return The values in the given field for each document.
* @throws IOException If any error occurs.
*/
- public DocTermsIndex getTermsIndex (AtomicReader reader, String field)
- throws IOException;
+ public SortedDocValues getTermsIndex (AtomicReader reader, String field) throws IOException;
/** Expert: just like {@link
* #getTermsIndex(AtomicReader,String)}, but you can specify
@@ -585,8 +659,7 @@ public interface FieldCache {
* faster lookups (default is "true"). Note that the
* first call for a given reader and field "wins",
* subsequent calls will share the same cache entry. */
- public DocTermsIndex getTermsIndex (AtomicReader reader, String field, float acceptableOverheadRatio)
- throws IOException;
+ public SortedDocValues getTermsIndex (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException;
/**
* Checks the internal cache for an appropriate entry, and if none is found, reads the term values
@@ -598,22 +671,51 @@ public interface FieldCache {
* @return a {@link DocTermOrds} instance
* @throws IOException If any error occurs.
*/
- public DocTermOrds getDocTermOrds(AtomicReader reader, String field) throws IOException;
+ public SortedSetDocValues getDocTermOrds(AtomicReader reader, String field) throws IOException;
/**
* EXPERT: A unique Identifier/Description for each item in the FieldCache.
* Can be useful for logging/debugging.
* @lucene.experimental
*/
- public static abstract class CacheEntry {
- public abstract Object getReaderKey();
- public abstract String getFieldName();
- public abstract Class<?> getCacheType();
- public abstract Object getCustom();
- public abstract Object getValue();
- private String size = null;
- protected final void setEstimatedSize(String size) {
- this.size = size;
+ public final class CacheEntry {
+
+ private final Object readerKey;
+ private final String fieldName;
+ private final Class<?> cacheType;
+ private final Object custom;
+ private final Object value;
+ private String size;
+
+ public CacheEntry(Object readerKey, String fieldName,
+ Class<?> cacheType,
+ Object custom,
+ Object value) {
+ this.readerKey = readerKey;
+ this.fieldName = fieldName;
+ this.cacheType = cacheType;
+ this.custom = custom;
+ this.value = value;
+ }
+
+ public Object getReaderKey() {
+ return readerKey;
+ }
+
+ public String getFieldName() {
+ return fieldName;
+ }
+
+ public Class<?> getCacheType() {
+ return cacheType;
+ }
+
+ public Object getCustom() {
+ return custom;
+ }
+
+ public Object getValue() {
+ return value;
}
/**
@@ -621,19 +723,18 @@ public interface FieldCache {
* @see #getEstimatedSize
*/
public void estimateSize() {
- long size = RamUsageEstimator.sizeOf(getValue());
- setEstimatedSize(RamUsageEstimator.humanReadableUnits(size));
+ long bytesUsed = RamUsageEstimator.sizeOf(getValue());
+ size = RamUsageEstimator.humanReadableUnits(bytesUsed);
}
/**
* The most recently estimated size of the value, null unless
* estimateSize has been called.
*/
- public final String getEstimatedSize() {
+ public String getEstimatedSize() {
return size;
}
-
@Override
public String toString() {
StringBuilder b = new StringBuilder();
@@ -650,9 +751,8 @@ public interface FieldCache {
return b.toString();
}
-
}
-
+
/**
* EXPERT: Generates an array of CacheEntry objects representing all items
* currently in the FieldCache.