You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/11/29 17:08:44 UTC
svn commit: r1415231 - in
/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene:
index/ search/
Author: mikemccand
Date: Thu Nov 29 16:08:41 2012
New Revision: 1415231
URL: http://svn.apache.org/viewvc?rev=1415231&view=rev
Log:
break DV/stored fields indexing into separate parts of indexing chain
Added:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java (with props)
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java (with props)
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java (with props)
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java (with props)
Modified:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java Thu Nov 29 16:08:41 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.codecs.BinaryDocValuesConsumer;
+import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefArray;
import org.apache.lucene.util.Counter;
@@ -29,7 +30,7 @@ import org.apache.lucene.util.Counter;
* segment flushes. */
// nocommit name?
// nocommit make this a consumer in the chain?
-class BytesDVWriter {
+class BytesDVWriter extends DocValuesWriter {
private final BytesRefArray bytesRefArray;
private final FieldInfo fieldInfo;
@@ -78,13 +79,18 @@ class BytesDVWriter {
totalSize += length;
}
+ @Override
public void finish(int maxDoc) {
if (addedValues < maxDoc) {
mergeLength(0);
}
}
- public void flush(FieldInfo fieldInfo, SegmentWriteState state, BinaryDocValuesConsumer consumer) throws IOException {
+ @Override
+ public void flush(SegmentWriteState state, SimpleDVConsumer dvConsumer) throws IOException {
+ BinaryDocValuesConsumer consumer = dvConsumer.addBinaryField(fieldInfo,
+ fixedLength >= 0,
+ maxLength);
final int bufferedDocCount = addedValues;
BytesRef value = new BytesRef();
for(int docID=0;docID<bufferedDocCount;docID++) {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java Thu Nov 29 16:08:41 2012
@@ -38,5 +38,4 @@ abstract class DocFieldConsumer {
abstract DocFieldConsumerPerField addField(FieldInfo fi);
abstract void finishDocument() throws IOException;
-
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java Thu Nov 29 16:08:41 2012
@@ -49,7 +49,7 @@ import org.apache.lucene.util.IOUtils;
final class DocFieldProcessor extends DocConsumer {
final DocFieldConsumer consumer;
- final StoredFieldsConsumer fieldsWriter;
+ final StoredFieldsConsumer storedConsumer;
final Codec codec;
// Holds all fields seen in current doc
@@ -66,12 +66,12 @@ final class DocFieldProcessor extends Do
final Counter bytesUsed;
- public DocFieldProcessor(DocumentsWriterPerThread docWriter, DocFieldConsumer consumer) {
+ public DocFieldProcessor(DocumentsWriterPerThread docWriter, DocFieldConsumer consumer, StoredFieldsConsumer storedConsumer) {
this.docState = docWriter.docState;
this.codec = docWriter.codec;
this.bytesUsed = docWriter.bytesUsed;
this.consumer = consumer;
- fieldsWriter = new StoredFieldsConsumer(docWriter);
+ this.storedConsumer = storedConsumer;
}
@Override
@@ -83,72 +83,9 @@ final class DocFieldProcessor extends Do
childFields.put(f.getFieldInfo().name, f);
}
- SimpleDVConsumer dvConsumer = null;
-
- for(int i=0;i<fieldHash.length;i++) {
- DocFieldProcessorPerField field = fieldHash[i];
- while(field != null) {
- // nocommit maybe we should sort by .... somethign?
- // field name? field number? else this is hash
- // order!!
- if (field.bytesDVWriter != null || field.numberDVWriter != null || field.sortedBytesDVWriter != null) {
-
- if (dvConsumer == null) {
- SimpleDocValuesFormat fmt = state.segmentInfo.getCodec().simpleDocValuesFormat();
- // nocommit once we make
- // Codec.simpleDocValuesFormat abstract, change
- // this to assert dvConsumer != null!
- if (fmt == null) {
- field = field.next;
- continue;
- }
-
- dvConsumer = fmt.fieldsConsumer(state);
- // nocommit shouldn't need null check:
- if (dvConsumer == null) {
- continue;
- }
- }
-
- if (field.bytesDVWriter != null) {
- field.bytesDVWriter.finish(state.segmentInfo.getDocCount());
- field.bytesDVWriter.flush(field.fieldInfo, state,
- dvConsumer.addBinaryField(field.fieldInfo,
- field.bytesDVWriter.fixedLength >= 0,
- field.bytesDVWriter.maxLength));
- // nocommit must null it out now else next seg
- // will flush even if no docs had DV...?
- }
-
- if (field.sortedBytesDVWriter != null) {
- field.sortedBytesDVWriter.finish(state.segmentInfo.getDocCount());
- field.sortedBytesDVWriter.flush(field.fieldInfo, state,
- dvConsumer.addSortedField(field.fieldInfo,
- field.sortedBytesDVWriter.hash.size(),
- field.sortedBytesDVWriter.fixedLength >= 0,
- field.sortedBytesDVWriter.maxLength));
- // nocommit must null it out now else next seg
- // will flush even if no docs had DV...?
- }
-
- if (field.numberDVWriter != null) {
- field.numberDVWriter.finish(state.segmentInfo.getDocCount());
- field.numberDVWriter.flush(field.fieldInfo, state,
- dvConsumer.addNumericField(field.fieldInfo,
- field.numberDVWriter.minValue,
- field.numberDVWriter.maxValue));
- // nocommit must null it out now else next seg
- // will flush even if no docs had DV...?
- }
- }
- field = field.next;
- }
- }
-
assert fields.size() == totalFieldCount;
-
- fieldsWriter.flush(state);
+ storedConsumer.flush(state);
consumer.flush(childFields, state);
for (DocValuesConsumerHolder consumer : docValues.values()) {
@@ -157,7 +94,7 @@ final class DocFieldProcessor extends Do
// close perDocConsumer during flush to ensure all files are flushed due to PerCodec CFS
// nocommit
- IOUtils.close(perDocConsumer, dvConsumer);
+ IOUtils.close(perDocConsumer);
// Important to save after asking consumer to flush so
// consumer can alter the FieldInfo* if necessary. EG,
@@ -188,7 +125,7 @@ final class DocFieldProcessor extends Do
// TODO add abort to PerDocConsumer!
try {
- fieldsWriter.abort();
+ storedConsumer.abort();
} catch (Throwable t) {
if (th == null) {
th = t;
@@ -278,7 +215,7 @@ final class DocFieldProcessor extends Do
public void processDocument(FieldInfos.Builder fieldInfos) throws IOException {
consumer.startDocument();
- fieldsWriter.startDocument();
+ storedConsumer.startDocument();
fieldCount = 0;
@@ -297,49 +234,17 @@ final class DocFieldProcessor extends Do
fp.addField(field);
}
+
for (StorableField field: docState.doc.storableFields()) {
final String fieldName = field.name();
IndexableFieldType ft = field.fieldType();
-
- DocFieldProcessorPerField fp = processField(fieldInfos, thisFieldGen, fieldName, ft);
- if (ft.stored()) {
- fieldsWriter.addField(field, fp.fieldInfo);
- }
+ FieldInfo fieldInfo = fieldInfos.addOrUpdate(fieldName, ft);
+ storedConsumer.addField(docState.docID, field, fieldInfo);
- // nocommit the DV indexing should be just another
- // consumer in the chain, not stuck inside here? this
- // source should just "dispatch"
final DocValues.Type dvType = ft.docValueType();
if (dvType != null) {
- switch(dvType) {
- case BYTES_VAR_STRAIGHT:
- case BYTES_FIXED_STRAIGHT:
- fp.addBytesDVField(docState.docID, field.binaryValue());
- break;
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_DEREF:
- case BYTES_FIXED_DEREF:
- fp.addSortedBytesDVField(docState.docID, field.binaryValue());
- break;
- case VAR_INTS:
- case FIXED_INTS_8:
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- fp.addNumberDVField(docState.docID, field.numericValue());
- break;
- case FLOAT_32:
- fp.addFloatDVField(docState.docID, field.numericValue());
- break;
- case FLOAT_64:
- fp.addDoubleDVField(docState.docID, field.numericValue());
- break;
- default:
- break;
- }
DocValuesConsumerHolder docValuesConsumer = docValuesConsumer(dvType,
- docState, fp.fieldInfo);
+ docState, fieldInfo);
DocValuesConsumer consumer = docValuesConsumer.docValuesConsumer;
if (docValuesConsumer.compatibility == null) {
consumer.add(docState.docID, field);
@@ -381,6 +286,7 @@ final class DocFieldProcessor extends Do
private DocFieldProcessorPerField processField(FieldInfos.Builder fieldInfos,
final int thisFieldGen, final String fieldName, IndexableFieldType ft) {
+
// Make sure we have a PerField allocated
final int hashPos = fieldName.hashCode() & hashMask;
DocFieldProcessorPerField fp = fieldHash[hashPos];
@@ -406,6 +312,9 @@ final class DocFieldProcessor extends Do
rehash();
}
} else {
+ // nocommit this is wasteful: it's another hash lookup
+ // by field name; can we just do fp.fieldInfo.update
+ // directly?
fieldInfos.addOrUpdate(fp.fieldInfo.name, ft);
}
@@ -436,7 +345,7 @@ final class DocFieldProcessor extends Do
@Override
void finishDocument() throws IOException {
try {
- fieldsWriter.finishDocument();
+ storedConsumer.finishDocument();
} finally {
consumer.finishDocument();
}
@@ -485,5 +394,4 @@ final class DocFieldProcessor extends Do
docValues.put(fieldInfo.name, docValuesConsumerAndDocID);
return docValuesConsumerAndDocID;
}
-
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java Thu Nov 29 16:08:41 2012
@@ -35,12 +35,6 @@ final class DocFieldProcessorPerField {
final FieldInfo fieldInfo;
private final Counter bytesUsed;
- // nocommit after flush we should null these out? then we
- // don't need reset() impl'd in each...
- BytesDVWriter bytesDVWriter;
- SortedBytesDVWriter sortedBytesDVWriter;
- NumberDVWriter numberDVWriter;
-
DocFieldProcessorPerField next;
int lastGen = -1;
@@ -54,58 +48,6 @@ final class DocFieldProcessorPerField {
this.bytesUsed = docFieldProcessor.bytesUsed;
}
- // nocommit make this generic chain through consumer?
- public void addBytesDVField(int docID, BytesRef value) {
- if (bytesDVWriter == null) {
- verifyField(fieldInfo, "binary");
- bytesDVWriter = new BytesDVWriter(fieldInfo, bytesUsed);
- }
- bytesDVWriter.addValue(docID, value);
- }
-
- // nocommit make this generic chain through consumer?
- public void addSortedBytesDVField(int docID, BytesRef value) {
- if (sortedBytesDVWriter == null) {
- verifyField(fieldInfo, "sorted");
- sortedBytesDVWriter = new SortedBytesDVWriter(fieldInfo, bytesUsed);
- }
- sortedBytesDVWriter.addValue(docID, value);
- }
-
- // nocommit make this generic chain through consumer?
- public void addNumberDVField(int docID, Number value) {
- if (numberDVWriter == null) {
- verifyField(fieldInfo, "numeric");
- numberDVWriter = new NumberDVWriter(fieldInfo, bytesUsed);
- }
- numberDVWriter.addValue(docID, value.longValue());
- }
-
- // nocommit make this generic chain through consumer?
- public void addFloatDVField(int docID, Number value) {
- if (numberDVWriter == null) {
- verifyField(fieldInfo, "numeric");
- numberDVWriter = new NumberDVWriter(fieldInfo, bytesUsed);
- }
- numberDVWriter.addValue(docID, Float.floatToRawIntBits(value.floatValue()));
- }
-
- // nocommit make this generic chain through consumer?
- public void addDoubleDVField(int docID, Number value) {
- if (numberDVWriter == null) {
- verifyField(fieldInfo, "numeric");
- numberDVWriter = new NumberDVWriter(fieldInfo, bytesUsed);
- }
- numberDVWriter.addValue(docID, Double.doubleToRawLongBits(value.doubleValue()));
- }
-
- private void verifyField(FieldInfo field, String type) {
- if (dvFields.containsKey(field)) {
- throw new IllegalArgumentException("Incompatible DocValues type: field \"" + field.name + "\" changed from " + dvFields.get(field) + " to " + type);
- }
- dvFields.put(field, type);
- }
-
public void addField(IndexableField field) {
if (fieldCount == fields.length) {
int newSize = ArrayUtil.oversize(fieldCount + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
@@ -119,14 +61,5 @@ final class DocFieldProcessorPerField {
public void abort() {
consumer.abort();
- if (bytesDVWriter != null) {
- bytesDVWriter.abort();
- }
- if (sortedBytesDVWriter != null) {
- sortedBytesDVWriter.abort();
- }
- if (numberDVWriter != null) {
- numberDVWriter.abort();
- }
}
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocInverter.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocInverter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocInverter.java Thu Nov 29 16:08:41 2012
@@ -77,6 +77,8 @@ final class DocInverter extends DocField
}
}
+ // nocommit nuke all freeRAMs: they are unused?
+
@Override
public boolean freeRAM() {
return consumer.freeRAM();
Added: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java?rev=1415231&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java Thu Nov 29 16:08:41 2012
@@ -0,0 +1,200 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.codecs.SimpleDVConsumer;
+import org.apache.lucene.codecs.SimpleDocValuesFormat;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Counter;
+
+final class DocValuesProcessor extends StoredFieldsConsumer {
+
+ // nocommit wasteful we also keep a map ... double the
+ // hash lookups ... would be better if DFP had "the one map"?
+ private final Map<String,DocValuesWriter> writers = new HashMap<String,DocValuesWriter>();
+ private final Counter bytesUsed;
+
+ public DocValuesProcessor(Counter bytesUsed) {
+ this.bytesUsed = bytesUsed;
+ }
+
+ @Override
+ void startDocument() {
+ }
+
+ @Override
+ void finishDocument() {
+ }
+
+ @Override
+ public void addField(int docID, StorableField field, FieldInfo fieldInfo) {
+ final DocValues.Type dvType = field.fieldType().docValueType();
+ if (dvType != null) {
+ switch(dvType) {
+ case BYTES_VAR_STRAIGHT:
+ case BYTES_FIXED_STRAIGHT:
+ addBinaryField(fieldInfo, docID, field.binaryValue());
+ break;
+ case BYTES_VAR_SORTED:
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_DEREF:
+ case BYTES_FIXED_DEREF:
+ addSortedField(fieldInfo, docID, field.binaryValue());
+ break;
+ case VAR_INTS:
+ case FIXED_INTS_8:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ addNumericField(fieldInfo, docID, field.numericValue().longValue());
+ break;
+ case FLOAT_32:
+ addNumericField(fieldInfo, docID, field.numericValue().floatValue());
+ break;
+ case FLOAT_64:
+ addNumericField(fieldInfo, docID, field.numericValue().doubleValue());
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ @Override
+ void flush(SegmentWriteState state) throws IOException {
+ if (!writers.isEmpty()) {
+ SimpleDocValuesFormat fmt = state.segmentInfo.getCodec().simpleDocValuesFormat();
+ // nocommit once we make
+ // Codec.simpleDocValuesFormat abstract, change
+ // this to assert fmt != null!
+ if (fmt == null) {
+ return;
+ }
+
+ SimpleDVConsumer dvConsumer = fmt.fieldsConsumer(state);
+ // nocommit change to assert != null:
+ if (dvConsumer == null) {
+ return;
+ }
+
+ for(DocValuesWriter writer : writers.values()) {
+ writer.finish(state.segmentInfo.getDocCount());
+ writer.flush(state, dvConsumer);
+ }
+
+ writers.clear();
+ dvConsumer.close();
+ }
+ }
+
+ void addBinaryField(FieldInfo fieldInfo, int docID, BytesRef value) {
+ DocValuesWriter writer = writers.get(fieldInfo.name);
+ BytesDVWriter binaryWriter;
+ if (writer == null) {
+ binaryWriter = new BytesDVWriter(fieldInfo, bytesUsed);
+ writers.put(fieldInfo.name, binaryWriter);
+ } else if (!(writer instanceof BytesDVWriter)) {
+ throw new IllegalArgumentException("Incompatible DocValues type: field \"" + fieldInfo.name + "\" changed from " + getTypeDesc(writer) + " to binary");
+ } else {
+ binaryWriter = (BytesDVWriter) writer;
+ }
+ binaryWriter.addValue(docID, value);
+ }
+
+ void addSortedField(FieldInfo fieldInfo, int docID, BytesRef value) {
+ DocValuesWriter writer = writers.get(fieldInfo.name);
+ SortedBytesDVWriter sortedWriter;
+ if (writer == null) {
+ sortedWriter = new SortedBytesDVWriter(fieldInfo, bytesUsed);
+ writers.put(fieldInfo.name, sortedWriter);
+ } else if (!(writer instanceof SortedBytesDVWriter)) {
+ throw new IllegalArgumentException("Incompatible DocValues type: field \"" + fieldInfo.name + "\" changed from " + getTypeDesc(writer) + " to sorted");
+ } else {
+ sortedWriter = (SortedBytesDVWriter) writer;
+ }
+ sortedWriter.addValue(docID, value);
+ }
+
+ void addNumericField(FieldInfo fieldInfo, int docID, long value) {
+ DocValuesWriter writer = writers.get(fieldInfo.name);
+ NumberDVWriter numericWriter;
+ if (writer == null) {
+ numericWriter = new NumberDVWriter(fieldInfo, bytesUsed);
+ writers.put(fieldInfo.name, numericWriter);
+ } else if (!(writer instanceof NumberDVWriter)) {
+ throw new IllegalArgumentException("Incompatible DocValues type: field \"" + fieldInfo.name + "\" changed from " + getTypeDesc(writer) + " to numeric");
+ } else {
+ numericWriter = (NumberDVWriter) writer;
+ }
+ numericWriter.addValue(docID, value);
+ }
+
+ void addNumericField(FieldInfo fieldInfo, int docID, float value) {
+ DocValuesWriter writer = writers.get(fieldInfo.name);
+ NumberDVWriter numericWriter;
+ if (writer == null) {
+ numericWriter = new NumberDVWriter(fieldInfo, bytesUsed);
+ writers.put(fieldInfo.name, numericWriter);
+ } else if (!(writer instanceof NumberDVWriter)) {
+ throw new IllegalArgumentException("Incompatible DocValues type: field \"" + fieldInfo.name + "\" changed from " + getTypeDesc(writer) + " to numeric");
+ } else {
+ numericWriter = (NumberDVWriter) writer;
+ }
+ numericWriter.addValue(docID, Float.floatToRawIntBits(value));
+ }
+
+ void addNumericField(FieldInfo fieldInfo, int docID, double value) {
+ DocValuesWriter writer = writers.get(fieldInfo.name);
+ NumberDVWriter numericWriter;
+ if (writer == null) {
+ numericWriter = new NumberDVWriter(fieldInfo, bytesUsed);
+ writers.put(fieldInfo.name, numericWriter);
+ } else if (!(writer instanceof NumberDVWriter)) {
+ throw new IllegalArgumentException("Incompatible DocValues type: field \"" + fieldInfo.name + "\" changed from " + getTypeDesc(writer) + " to numeric");
+ } else {
+ numericWriter = (NumberDVWriter) writer;
+ }
+ numericWriter.addValue(docID, Double.doubleToRawLongBits(value));
+ }
+
+ private String getTypeDesc(DocValuesWriter obj) {
+ if (obj instanceof BytesDVWriter) {
+ return "binary";
+ } else if (obj instanceof NumberDVWriter) {
+ return "numeric";
+ } else {
+ assert obj instanceof SortedBytesDVWriter;
+ return "sorted";
+ }
+ }
+
+ @Override
+ public void abort() throws IOException {
+ for(DocValuesWriter writer : writers.values()) {
+ try {
+ writer.abort();
+ } catch (Throwable t) {
+ }
+ }
+ writers.clear();
+ }
+}
Added: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java?rev=1415231&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java Thu Nov 29 16:08:41 2012
@@ -0,0 +1,28 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.SimpleDVConsumer;
+
+abstract class DocValuesWriter {
+ abstract void abort() throws IOException;
+ abstract void finish(int numDoc);
+ abstract void flush(SegmentWriteState state, SimpleDVConsumer consumer) throws IOException;
+}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Thu Nov 29 16:08:41 2012
@@ -63,18 +63,21 @@ class DocumentsWriterPerThread {
This is the current indexing chain:
DocConsumer / DocConsumerPerThread
- --> code: DocFieldProcessor / DocFieldProcessorPerThread
- --> DocFieldConsumer / DocFieldConsumerPerThread / DocFieldConsumerPerField
- --> code: DocFieldConsumers / DocFieldConsumersPerThread / DocFieldConsumersPerField
- --> code: DocInverter / DocInverterPerThread / DocInverterPerField
- --> InvertedDocConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
- --> code: TermsHash / TermsHashPerThread / TermsHashPerField
- --> TermsHashConsumer / TermsHashConsumerPerThread / TermsHashConsumerPerField
- --> code: FreqProxTermsWriter / FreqProxTermsWriterPerThread / FreqProxTermsWriterPerField
- --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerThread / TermVectorsTermsWriterPerField
- --> InvertedDocEndConsumer / InvertedDocConsumerPerThread / InvertedDocConsumerPerField
- --> code: NormsWriter / NormsWriterPerThread / NormsWriterPerField
- --> code: StoredFieldsWriter / StoredFieldsWriterPerThread / StoredFieldsWriterPerField
+ --> code: DocFieldProcessor
+ --> DocFieldConsumer / DocFieldConsumerPerField
+ --> code: DocFieldConsumers / DocFieldConsumersPerField
+ --> code: DocInverter / DocInverterPerField
+ --> InvertedDocConsumer / InvertedDocConsumerPerField
+ --> code: TermsHash / TermsHashPerField
+ --> TermsHashConsumer / TermsHashConsumerPerField
+ --> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
+ --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
+ --> InvertedDocEndConsumer / InvertedDocConsumerPerField
+ --> code: NormsWriter / NormsWriterPerField
+ --> StoredFieldsConsumer
+ --> TwoStoredFieldConsumers
+ -> code: StoredFieldsProcessor
+ -> code: DocValuesProcessor
*/
// Build up indexing chain:
@@ -86,7 +89,10 @@ class DocumentsWriterPerThread {
new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
final NormsConsumer normsWriter = new NormsConsumer(documentsWriterPerThread);
final DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
- return new DocFieldProcessor(documentsWriterPerThread, docInverter);
+ final StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(
+ new StoredFieldsProcessor(documentsWriterPerThread),
+ new DocValuesProcessor(documentsWriterPerThread.bytesUsed));
+ return new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields);
}
};
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumberDVWriter.java Thu Nov 29 16:08:41 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.codecs.NumericDocValuesConsumer;
+import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.packed.AppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
@@ -30,7 +31,7 @@ import org.apache.lucene.util.packed.Pac
* segment flushes. */
// nocommit name?
// nocommit make this a consumer in the chain?
-class NumberDVWriter {
+class NumberDVWriter extends DocValuesWriter {
private final static long MISSING = 0L;
@@ -83,13 +84,16 @@ class NumberDVWriter {
}
}
+ @Override
public void finish(int maxDoc) {
if (pending.size() < maxDoc) {
mergeValue(0);
}
}
- public void flush(FieldInfo fieldInfo, SegmentWriteState state, NumericDocValuesConsumer consumer) throws IOException {
+ @Override
+ public void flush(SegmentWriteState state, SimpleDVConsumer dvConsumer) throws IOException {
+ NumericDocValuesConsumer consumer = dvConsumer.addNumericField(fieldInfo, minValue, maxValue);
final int bufferedDocCount = pending.size();
AppendingLongBuffer.Iterator it = pending.iterator();
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedBytesDVWriter.java Thu Nov 29 16:08:41 2012
@@ -19,21 +19,22 @@ package org.apache.lucene.index;
import java.io.IOException;
+import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SortedDocValuesConsumer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
/** Buffers up pending byte[] per doc, deref and sorting via
* int ord, then flushes when segment flushes. */
// nocommit name?
// nocommit make this a consumer in the chain?
-class SortedBytesDVWriter {
+class SortedBytesDVWriter extends DocValuesWriter {
final BytesRefHash hash;
private int[] pending = new int[DEFAULT_PENDING_SIZE];
private int pendingIndex = 0;
@@ -76,6 +77,7 @@ class SortedBytesDVWriter {
addOneValue(value);
}
+ @Override
public void finish(int maxDoc) {
if (pendingIndex < maxDoc) {
addOneValue(EMPTY);
@@ -107,8 +109,12 @@ class SortedBytesDVWriter {
maxLength = Math.max(maxLength, length);
}
- public void flush(FieldInfo fieldInfo, SegmentWriteState state, SortedDocValuesConsumer consumer) throws IOException {
-
+ @Override
+ public void flush(SegmentWriteState state, SimpleDVConsumer dvConsumer) throws IOException {
+ SortedDocValuesConsumer consumer = dvConsumer.addSortedField(fieldInfo,
+ hash.size(),
+ fixedLength >= 0,
+ maxLength);
final int maxDoc = state.segmentInfo.getDocCount();
int emptyOrd = -1;
if (pendingIndex < maxDoc) {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java Thu Nov 29 16:08:41 2012
@@ -19,129 +19,10 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.StoredFieldsWriter;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.RamUsageEstimator;
-
-/** This is a DocFieldConsumer that writes stored fields. */
-final class StoredFieldsConsumer {
-
- StoredFieldsWriter fieldsWriter;
- final DocumentsWriterPerThread docWriter;
- int lastDocID;
-
- int freeCount;
-
- final DocumentsWriterPerThread.DocState docState;
- final Codec codec;
-
- public StoredFieldsConsumer(DocumentsWriterPerThread docWriter) {
- this.docWriter = docWriter;
- this.docState = docWriter.docState;
- this.codec = docWriter.codec;
- }
-
- private int numStoredFields;
- private StorableField[] storedFields;
- private FieldInfo[] fieldInfos;
-
- public void reset() {
- numStoredFields = 0;
- storedFields = new StorableField[1];
- fieldInfos = new FieldInfo[1];
- }
-
- public void startDocument() {
- reset();
- }
-
- public void flush(SegmentWriteState state) throws IOException {
- int numDocs = state.segmentInfo.getDocCount();
-
- if (numDocs > 0) {
- // It's possible that all documents seen in this segment
- // hit non-aborting exceptions, in which case we will
- // not have yet init'd the FieldsWriter:
- initFieldsWriter(state.context);
- fill(numDocs);
- }
-
- if (fieldsWriter != null) {
- try {
- fieldsWriter.finish(state.fieldInfos, numDocs);
- } finally {
- fieldsWriter.close();
- fieldsWriter = null;
- lastDocID = 0;
- }
- }
- }
-
- private synchronized void initFieldsWriter(IOContext context) throws IOException {
- if (fieldsWriter == null) {
- fieldsWriter = codec.storedFieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
- lastDocID = 0;
- }
- }
-
- int allocCount;
-
- void abort() {
- reset();
-
- if (fieldsWriter != null) {
- fieldsWriter.abort();
- fieldsWriter = null;
- lastDocID = 0;
- }
- }
-
- /** Fills in any hole in the docIDs */
- void fill(int docID) throws IOException {
- // We must "catch up" for all docs before us
- // that had no stored fields:
- while(lastDocID < docID) {
- fieldsWriter.startDocument(0);
- lastDocID++;
- }
- }
-
- void finishDocument() throws IOException {
- assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument start");
-
- initFieldsWriter(IOContext.DEFAULT);
- fill(docState.docID);
-
- if (fieldsWriter != null && numStoredFields > 0) {
- fieldsWriter.startDocument(numStoredFields);
- for (int i = 0; i < numStoredFields; i++) {
- fieldsWriter.writeField(fieldInfos[i], storedFields[i]);
- }
- lastDocID++;
- }
-
- reset();
- assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end");
- }
-
- public void addField(StorableField field, FieldInfo fieldInfo) {
- if (numStoredFields == storedFields.length) {
- int newSize = ArrayUtil.oversize(numStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- StorableField[] newArray = new StorableField[newSize];
- System.arraycopy(storedFields, 0, newArray, 0, numStoredFields);
- storedFields = newArray;
-
- FieldInfo[] newInfoArray = new FieldInfo[newSize];
- System.arraycopy(fieldInfos, 0, newInfoArray, 0, numStoredFields);
- fieldInfos = newInfoArray;
- }
-
- storedFields[numStoredFields] = field;
- fieldInfos[numStoredFields] = fieldInfo;
- numStoredFields++;
-
- assert docState.testPoint("StoredFieldsWriterPerThread.processFields.writeField");
- }
+abstract class StoredFieldsConsumer {
+ abstract void addField(int docID, StorableField field, FieldInfo fieldInfo) throws IOException;
+ abstract void flush(SegmentWriteState state) throws IOException;
+ abstract void abort() throws IOException;
+ abstract void startDocument() throws IOException;
+ abstract void finishDocument() throws IOException;
}
Added: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java?rev=1415231&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java Thu Nov 29 16:08:41 2012
@@ -0,0 +1,154 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.RamUsageEstimator;
+
+/** This is a StoredFieldsConsumer that writes stored fields. */
+final class StoredFieldsProcessor extends StoredFieldsConsumer {
+
+ StoredFieldsWriter fieldsWriter;
+ final DocumentsWriterPerThread docWriter;
+ int lastDocID;
+
+ int freeCount;
+
+ final DocumentsWriterPerThread.DocState docState;
+ final Codec codec;
+
+ public StoredFieldsProcessor(DocumentsWriterPerThread docWriter) {
+ this.docWriter = docWriter;
+ this.docState = docWriter.docState;
+ this.codec = docWriter.codec;
+ }
+
+ private int numStoredFields;
+ private StorableField[] storedFields;
+ private FieldInfo[] fieldInfos;
+
+ public void reset() {
+ numStoredFields = 0;
+ storedFields = new StorableField[1];
+ fieldInfos = new FieldInfo[1];
+ }
+
+ @Override
+ public void startDocument() {
+ reset();
+ }
+
+ @Override
+ public void flush(SegmentWriteState state) throws IOException {
+ int numDocs = state.segmentInfo.getDocCount();
+
+ if (numDocs > 0) {
+ // It's possible that all documents seen in this segment
+ // hit non-aborting exceptions, in which case we will
+ // not have yet init'd the FieldsWriter:
+ initFieldsWriter(state.context);
+ fill(numDocs);
+ }
+
+ if (fieldsWriter != null) {
+ try {
+ fieldsWriter.finish(state.fieldInfos, numDocs);
+ } finally {
+ fieldsWriter.close();
+ fieldsWriter = null;
+ lastDocID = 0;
+ }
+ }
+ }
+
+ private synchronized void initFieldsWriter(IOContext context) throws IOException {
+ if (fieldsWriter == null) {
+ fieldsWriter = codec.storedFieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
+ lastDocID = 0;
+ }
+ }
+
+ int allocCount;
+
+ @Override
+ void abort() {
+ reset();
+
+ if (fieldsWriter != null) {
+ fieldsWriter.abort();
+ fieldsWriter = null;
+ lastDocID = 0;
+ }
+ }
+
+ /** Fills in any hole in the docIDs */
+ void fill(int docID) throws IOException {
+ // We must "catch up" for all docs before us
+ // that had no stored fields:
+ while(lastDocID < docID) {
+ fieldsWriter.startDocument(0);
+ lastDocID++;
+ }
+ }
+
+ @Override
+ void finishDocument() throws IOException {
+ assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument start");
+
+ initFieldsWriter(IOContext.DEFAULT);
+ fill(docState.docID);
+
+ if (fieldsWriter != null && numStoredFields > 0) {
+ fieldsWriter.startDocument(numStoredFields);
+ for (int i = 0; i < numStoredFields; i++) {
+ fieldsWriter.writeField(fieldInfos[i], storedFields[i]);
+ }
+ lastDocID++;
+ }
+
+ reset();
+ assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end");
+ }
+
+ @Override
+ public void addField(int docID, StorableField field, FieldInfo fieldInfo) {
+ if (field.fieldType().stored()) {
+ if (numStoredFields == storedFields.length) {
+ int newSize = ArrayUtil.oversize(numStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
+ StorableField[] newArray = new StorableField[newSize];
+ System.arraycopy(storedFields, 0, newArray, 0, numStoredFields);
+ storedFields = newArray;
+
+ FieldInfo[] newInfoArray = new FieldInfo[newSize];
+ System.arraycopy(fieldInfos, 0, newInfoArray, 0, numStoredFields);
+ fieldInfos = newInfoArray;
+ }
+
+ storedFields[numStoredFields] = field;
+ fieldInfos[numStoredFields] = fieldInfo;
+ numStoredFields++;
+
+ assert docState.testPoint("StoredFieldsWriterPerThread.processFields.writeField");
+ }
+ }
+}
Added: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java?rev=1415231&view=auto
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java (added)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java Thu Nov 29 16:08:41 2012
@@ -0,0 +1,73 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+/** Just switches between two {@link DocFieldConsumer}s. */
+
+class TwoStoredFieldsConsumers extends StoredFieldsConsumer {
+ private final StoredFieldsConsumer first;
+ private final StoredFieldsConsumer second;
+
+ public TwoStoredFieldsConsumers(StoredFieldsConsumer first, StoredFieldsConsumer second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ @Override
+ public void addField(int docID, StorableField field, FieldInfo fieldInfo) throws IOException {
+ first.addField(docID, field, fieldInfo);
+ second.addField(docID, field, fieldInfo);
+ }
+
+ @Override
+ void flush(SegmentWriteState state) throws IOException {
+ first.flush(state);
+ second.flush(state);
+ }
+
+ @Override
+ void abort() {
+ try {
+ first.abort();
+ } catch (Throwable t) {
+ }
+ try {
+ second.abort();
+ } catch (Throwable t) {
+ }
+ }
+
+ @Override
+ void startDocument() throws IOException {
+ first.startDocument();
+ second.startDocument();
+ }
+
+ @Override
+ void finishDocument() throws IOException {
+ // nocommit must this be a try/finally...? i'd prefer
+ // not ...
+ try {
+ first.finishDocument();
+ } finally {
+ second.finishDocument();
+ }
+ }
+}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1415231&r1=1415230&r2=1415231&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Thu Nov 29 16:08:41 2012
@@ -1328,7 +1328,7 @@ class FieldCacheImpl implements FieldCac
BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
if (valuesIn != null) {
- return valuesIn;
+ return valuesIn.newRAMInstance();
} else {
final int maxDoc = reader.maxDoc();
Terms terms = reader.terms(key.field);