You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/12/07 15:29:11 UTC
svn commit: r1211451 - in
/lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index:
SegmentMerger.java codecs/PerDocConsumer.java
Author: rmuir
Date: Wed Dec 7 14:29:11 2011
New Revision: 1211451
URL: http://svn.apache.org/viewvc?rev=1211451&view=rev
Log:
LUCENE-3622: merge docvalues fieldsinfos in mergeFieldInfos
Modified:
lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java
Modified: lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1211451&r1=1211450&r2=1211451&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Wed Dec 7 14:29:11 2011
@@ -21,7 +21,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader.FieldOption;
@@ -31,6 +33,8 @@ import org.apache.lucene.index.codecs.Fi
import org.apache.lucene.index.codecs.StoredFieldsWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.TermVectorsWriter;
+import org.apache.lucene.index.values.IndexDocValues;
+import org.apache.lucene.index.values.TypePromoter;
import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -128,9 +132,7 @@ final class SegmentMerger {
int numMerged = mergeVectors();
assert numMerged == mergeState.mergedDocCount;
}
- // write FIS once merge is done. IDV might change types or drops fields
- FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
- fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
+
return mergeState;
}
@@ -183,15 +185,40 @@ final class SegmentMerger {
}
}
}
+
+ // returns an updated typepromoter (tracking type and size) given a previous one,
+ // and a newly encountered docvalues
+ private TypePromoter mergeDocValuesType(TypePromoter previous, IndexDocValues docValues) {
+ TypePromoter incoming = TypePromoter.create(docValues.type(), docValues.getValueSize());
+ if (previous == null) {
+ previous = TypePromoter.getIdentityPromoter();
+ }
+ TypePromoter promoted = previous.promote(incoming);
+ if (promoted == null) {
+ // type is incompatible: promote to BYTES_VAR_STRAIGHT
+ return TypePromoter.create(ValueType.BYTES_VAR_STRAIGHT, TypePromoter.VAR_TYPE_VALUE_SIZE);
+ } else {
+ return promoted;
+ }
+ }
private void mergeFieldInfos() throws IOException {
+ // mapping from all docvalues fields found to their promoted types
+ // this is because FieldInfos does not store the valueSize
+ Map<FieldInfo,TypePromoter> docValuesTypes = new HashMap<FieldInfo,TypePromoter>();
+
for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) {
final IndexReader reader = readerAndLiveDocs.reader;
if (reader instanceof SegmentReader) {
SegmentReader segmentReader = (SegmentReader) reader;
FieldInfos readerFieldInfos = segmentReader.fieldInfos();
for (FieldInfo fi : readerFieldInfos) {
- mergeState.fieldInfos.add(fi);
+ FieldInfo merged = mergeState.fieldInfos.add(fi);
+ // update the type promotion mapping for this reader
+ if (fi.hasDocValues()) {
+ TypePromoter previous = docValuesTypes.get(merged);
+ docValuesTypes.put(merged, mergeDocValuesType(previous, reader.docValues(fi.name)));
+ }
}
} else {
addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
@@ -206,10 +233,33 @@ final class SegmentMerger {
Collection<String> dvNames = reader.getFieldNames(FieldOption.DOC_VALUES);
mergeState.fieldInfos.addOrUpdate(dvNames, false);
for (String dvName : dvNames) {
- mergeState.fieldInfos.fieldInfo(dvName).setDocValues(reader.docValues(dvName).type());
+ FieldInfo merged = mergeState.fieldInfos.fieldInfo(dvName);
+ IndexDocValues docValues = reader.docValues(dvName);
+ merged.setDocValues(docValues.type());
+ TypePromoter previous = docValuesTypes.get(merged);
+ docValuesTypes.put(merged, mergeDocValuesType(previous, docValues));
}
}
}
+
+ // update any promoted doc values types:
+ for (Map.Entry<FieldInfo,TypePromoter> e : docValuesTypes.entrySet()) {
+ FieldInfo fi = e.getKey();
+ TypePromoter promoter = e.getValue();
+ if (promoter == null) {
+ fi.resetDocValues(null);
+ } else {
+ assert promoter != TypePromoter.getIdentityPromoter();
+ if (fi.getDocValues() != promoter.type()) {
+ // reset the type if we got promoted
+ fi.resetDocValues(promoter.type());
+ }
+ }
+ }
+
+ // write the merged infos
+ FieldInfosWriter fieldInfosWriter = codec.fieldInfosFormat().getFieldInfosWriter();
+ fieldInfosWriter.write(directory, segment, mergeState.fieldInfos, context);
}
/**
Modified: lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java?rev=1211451&r1=1211450&r2=1211451&view=diff
==============================================================================
--- lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java (original)
+++ lucene/dev/branches/lucene3622/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java Wed Dec 7 14:29:11 2011
@@ -19,11 +19,8 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.TypePromoter;
-import org.apache.lucene.index.values.ValueType;
/**
* Abstract API that consumes per document values. Concrete implementations of
@@ -44,75 +41,27 @@ public abstract class PerDocConsumer imp
* Consumes and merges the given {@link PerDocValues} producer
* into this consumers format.
*/
- public void merge(MergeState mergeState)
- throws IOException {
- final FieldInfos fieldInfos = mergeState.fieldInfos;
+ public void merge(MergeState mergeState) throws IOException {
final IndexDocValues[] docValues = new IndexDocValues[mergeState.readers.size()];
final PerDocValues[] perDocValues = new PerDocValues[mergeState.readers.size()];
// pull all PerDocValues
for (int i = 0; i < perDocValues.length; i++) {
- perDocValues[i] = mergeState.readers.get(i).reader.perDocValues();
+ perDocValues[i] = mergeState.readers.get(i).reader.perDocValues();
}
- for (FieldInfo fieldInfo : fieldInfos) {
- mergeState.fieldInfo = fieldInfo;
- TypePromoter currentPromoter = TypePromoter.getIdentityPromoter();
+ for (FieldInfo fieldInfo : mergeState.fieldInfos) {
+ mergeState.fieldInfo = fieldInfo; // set the field we are merging
if (fieldInfo.hasDocValues()) {
for (int i = 0; i < perDocValues.length; i++) {
if (perDocValues[i] != null) { // get all IDV to merge
docValues[i] = perDocValues[i].docValues(fieldInfo.name);
- if (docValues[i] != null) {
- currentPromoter = promoteValueType(fieldInfo, docValues[i], currentPromoter);
- if (currentPromoter == null) {
- break;
- }
- }
}
}
-
- if (currentPromoter == null) {
- fieldInfo.resetDocValues(null);
- continue;
- }
- assert currentPromoter != TypePromoter.getIdentityPromoter();
- if (fieldInfo.getDocValues() != currentPromoter.type()) {
- // reset the type if we got promoted
- fieldInfo.resetDocValues(currentPromoter.type());
- }
-
- final DocValuesConsumer docValuesConsumer = addValuesField(mergeState.fieldInfo);
+ final DocValuesConsumer docValuesConsumer = addValuesField(fieldInfo);
assert docValuesConsumer != null;
docValuesConsumer.merge(mergeState, docValues);
}
}
/* NOTE: don't close the perDocProducers here since they are private segment producers
* and will be closed once the SegmentReader goes out of scope */
- }
-
- protected TypePromoter promoteValueType(final FieldInfo fieldInfo, final IndexDocValues docValues,
- TypePromoter currentPromoter) {
- assert currentPromoter != null;
- final TypePromoter incomingPromoter = TypePromoter.create(docValues.type(), docValues.getValueSize());
- assert incomingPromoter != null;
- final TypePromoter newPromoter = currentPromoter.promote(incomingPromoter);
- return newPromoter == null ? handleIncompatibleValueType(fieldInfo, incomingPromoter, currentPromoter) : newPromoter;
- }
-
- /**
- * Resolves a conflicts of incompatible {@link TypePromoter}s. The default
- * implementation promotes incompatible types to
- * {@link ValueType#BYTES_VAR_STRAIGHT} and preserves all values. If this
- * method returns <code>null</code> all docvalues for the given
- * {@link FieldInfo} are dropped and all values are lost.
- *
- * @param incomingPromoter
- * the incompatible incoming promoter
- * @param currentPromoter
- * the current promoter
- * @return a promoted {@link TypePromoter} or <code>null</code> iff this index
- * docvalues should be dropped for this field.
- */
- protected TypePromoter handleIncompatibleValueType(FieldInfo fieldInfo, TypePromoter incomingPromoter, TypePromoter currentPromoter) {
- return TypePromoter.create(ValueType.BYTES_VAR_STRAIGHT, TypePromoter.VAR_TYPE_VALUE_SIZE);
- }
-
+ }
}