You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/05 16:01:17 UTC
svn commit: r1629499 [5/7] - in /lucene/dev/trunk: ./ dev-tools/ lucene/
lucene/backward-codecs/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/
lucene/backward-codec...
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java Sun Oct 5 14:01:13 2014
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.apache.lucene.index.CorruptIndexException;
@@ -96,34 +97,46 @@ public final class CodecUtil {
* Writes a codec header for a per-segment, which records both a string to
* identify the file, a version number, and the unique ID of the segment.
* This header can be parsed and validated with
- * {@link #checkSegmentHeader(DataInput, String, int, int, byte[]) checkSegmentHeader()}.
+ * {@link #checkSegmentHeader(DataInput, String, int, int, byte[], String) checkSegmentHeader()}.
* <p>
- * CodecSegmentHeader --> CodecHeader,SegmentID
+ * CodecSegmentHeader --> CodecHeader,SegmentID,SegmentSuffix
* <ul>
- * <li>CodecHeader --> {@link #writeHeader}
- * <li>SegmentID --> {@link DataOutput#writeByte byte}<sup>16</sup>.
- * Unique identifier for the segment.
+ * <li>CodecHeader --> {@link #writeHeader}
+ * <li>SegmentID --> {@link DataOutput#writeByte byte}<sup>16</sup>
+ * <li>SegmentSuffix --> SuffixLength,SuffixBytes
+ * <li>SuffixLength --> {@link DataOutput#writeByte byte}
+ * <li>SuffixBytes --> {@link DataOutput#writeByte byte}<sup>SuffixLength</sup>
* </ul>
* <p>
* Note that the length of a segment header depends only upon the
- * name of the codec, so this length can be computed at any time
- * with {@link #headerLength(String)}.
+ * name of the codec and suffix, so this length can be computed at any time
+ * with {@link #segmentHeaderLength(String,String)}.
*
* @param out Output stream
* @param codec String to identify this file. It should be simple ASCII,
* less than 128 characters in length.
* @param segmentID Unique identifier for the segment
+ * @param segmentSuffix auxiliary suffix for the file. It should be simple ASCII,
+ * less than 256 characters in length.
* @param version Version number
* @throws IOException If there is an I/O error writing to the underlying medium.
* @throws IllegalArgumentException If the codec name is not simple ASCII, or
- * is more than 127 characters in length, or if segmentID is invalid.
+ * is more than 127 characters in length, or if segmentID is invalid,
+ * or if the segmentSuffix is not simple ASCII, or more than 255 characters
+ * in length.
*/
- public static void writeSegmentHeader(DataOutput out, String codec, int version, byte[] segmentID) throws IOException {
+ public static void writeSegmentHeader(DataOutput out, String codec, int version, byte[] segmentID, String segmentSuffix) throws IOException {
if (segmentID.length != StringHelper.ID_LENGTH) {
throw new IllegalArgumentException("Invalid id: " + StringHelper.idToString(segmentID));
}
writeHeader(out, codec, version);
out.writeBytes(segmentID, 0, segmentID.length);
+ BytesRef suffixBytes = new BytesRef(segmentSuffix);
+ if (suffixBytes.length != segmentSuffix.length() || suffixBytes.length >= 256) {
+ throw new IllegalArgumentException("codec must be simple ASCII, less than 256 characters in length [got " + segmentSuffix + "]");
+ }
+ out.writeByte((byte)suffixBytes.length);
+ out.writeBytes(suffixBytes.bytes, suffixBytes.offset, suffixBytes.length);
}
/**
@@ -142,10 +155,10 @@ public final class CodecUtil {
*
* @param codec Codec name.
* @return length of the entire segment header.
- * @see #writeSegmentHeader(DataOutput, String, int, byte[])
+ * @see #writeSegmentHeader(DataOutput, String, int, byte[], String)
*/
- public static int segmentHeaderLength(String codec) {
- return headerLength(codec) + StringHelper.ID_LENGTH;
+ public static int segmentHeaderLength(String codec, String segmentSuffix) {
+ return headerLength(codec) + StringHelper.ID_LENGTH + 1 + segmentSuffix.length();
}
/**
@@ -207,7 +220,7 @@ public final class CodecUtil {
/**
* Reads and validates a header previously written with
- * {@link #writeSegmentHeader(DataOutput, String, int, byte[])}.
+ * {@link #writeSegmentHeader(DataOutput, String, int, byte[], String)}.
* <p>
* When reading a file, supply the expected <code>codec</code>,
* expected version range (<code>minVersion to maxVersion</code>),
@@ -220,6 +233,7 @@ public final class CodecUtil {
* @param minVersion The minimum supported expected version number.
* @param maxVersion The maximum supported expected version number.
* @param segmentID The expected segment this file belongs to.
+ * @param segmentSuffix The expected auxiliary segment suffix for this file.
* @return The actual version found, when a valid header is found
* that matches <code>codec</code>, with an actual version
* where <code>minVersion <= actual <= maxVersion</code>,
@@ -228,15 +242,15 @@ public final class CodecUtil {
* @throws CorruptIndexException If the first four bytes are not
* {@link #CODEC_MAGIC}, or if the actual codec found is
* not <code>codec</code>, or if the <code>segmentID</code>
- * does not match.
+ * or <code>segmentSuffix</code> do not match.
* @throws IndexFormatTooOldException If the actual version is less
* than <code>minVersion</code>.
* @throws IndexFormatTooNewException If the actual version is greater
* than <code>maxVersion</code>.
* @throws IOException If there is an I/O error reading from the underlying medium.
- * @see #writeSegmentHeader(DataOutput, String, int, byte[])
+ * @see #writeSegmentHeader(DataOutput, String, int, byte[],String)
*/
- public static int checkSegmentHeader(DataInput in, String codec, int minVersion, int maxVersion, byte[] segmentID) throws IOException {
+ public static int checkSegmentHeader(DataInput in, String codec, int minVersion, int maxVersion, byte[] segmentID, String segmentSuffix) throws IOException {
int version = checkHeader(in, codec, minVersion, maxVersion);
byte id[] = new byte[StringHelper.ID_LENGTH];
in.readBytes(id, 0, id.length);
@@ -244,6 +258,14 @@ public final class CodecUtil {
throw new CorruptIndexException("file mismatch, expected segment id=" + StringHelper.idToString(segmentID)
+ ", got=" + StringHelper.idToString(id), in);
}
+ int suffixLength = in.readByte() & 0xFF;
+ byte suffixBytes[] = new byte[suffixLength];
+ in.readBytes(suffixBytes, 0, suffixBytes.length);
+ String suffix = new String(suffixBytes, 0, suffixBytes.length, StandardCharsets.UTF_8);
+ if (!suffix.equals(segmentSuffix)) {
+ throw new CorruptIndexException("file mismatch, expected segment suffix=" + segmentSuffix
+ + ", got=" + suffix, in);
+ }
return version;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java Sun Oct 5 14:01:13 2014
@@ -24,7 +24,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FilteredTermsEnum;
@@ -131,67 +130,110 @@ public abstract class DocValuesConsumer
* Implementations can override this method
* for more sophisticated merging (bulk-byte copying, etc). */
public void merge(MergeState mergeState) throws IOException {
- for (FieldInfo field : mergeState.fieldInfos) {
- DocValuesType type = field.getDocValuesType();
+ for(DocValuesProducer docValuesProducer : mergeState.docValuesProducers) {
+ if (docValuesProducer != null) {
+ docValuesProducer.checkIntegrity();
+ }
+ }
+
+ for (FieldInfo mergeFieldInfo : mergeState.mergeFieldInfos) {
+ DocValuesType type = mergeFieldInfo.getDocValuesType();
if (type != null) {
if (type == DocValuesType.NUMERIC) {
List<NumericDocValues> toMerge = new ArrayList<>();
List<Bits> docsWithField = new ArrayList<>();
- for (LeafReader reader : mergeState.readers) {
- NumericDocValues values = reader.getNumericDocValues(field.name);
- Bits bits = reader.getDocsWithField(field.name);
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ NumericDocValues values = null;
+ Bits bits = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.NUMERIC) {
+ values = docValuesProducer.getNumeric(fieldInfo);
+ bits = docValuesProducer.getDocsWithField(fieldInfo);
+ }
+ }
if (values == null) {
values = DocValues.emptyNumeric();
- bits = new Bits.MatchNoBits(reader.maxDoc());
+ bits = new Bits.MatchNoBits(mergeState.maxDocs[i]);
}
toMerge.add(values);
docsWithField.add(bits);
}
- mergeNumericField(field, mergeState, toMerge, docsWithField);
+ mergeNumericField(mergeFieldInfo, mergeState, toMerge, docsWithField);
} else if (type == DocValuesType.BINARY) {
List<BinaryDocValues> toMerge = new ArrayList<>();
List<Bits> docsWithField = new ArrayList<>();
- for (LeafReader reader : mergeState.readers) {
- BinaryDocValues values = reader.getBinaryDocValues(field.name);
- Bits bits = reader.getDocsWithField(field.name);
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ BinaryDocValues values = null;
+ Bits bits = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
+ values = docValuesProducer.getBinary(fieldInfo);
+ bits = docValuesProducer.getDocsWithField(fieldInfo);
+ }
+ }
if (values == null) {
values = DocValues.emptyBinary();
- bits = new Bits.MatchNoBits(reader.maxDoc());
+ bits = new Bits.MatchNoBits(mergeState.maxDocs[i]);
}
toMerge.add(values);
docsWithField.add(bits);
}
- mergeBinaryField(field, mergeState, toMerge, docsWithField);
+ mergeBinaryField(mergeFieldInfo, mergeState, toMerge, docsWithField);
} else if (type == DocValuesType.SORTED) {
List<SortedDocValues> toMerge = new ArrayList<>();
- for (LeafReader reader : mergeState.readers) {
- SortedDocValues values = reader.getSortedDocValues(field.name);
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ SortedDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED) {
+ values = docValuesProducer.getSorted(fieldInfo);
+ }
+ }
if (values == null) {
values = DocValues.emptySorted();
}
toMerge.add(values);
}
- mergeSortedField(field, mergeState, toMerge);
+ mergeSortedField(mergeFieldInfo, mergeState, toMerge);
} else if (type == DocValuesType.SORTED_SET) {
List<SortedSetDocValues> toMerge = new ArrayList<>();
- for (LeafReader reader : mergeState.readers) {
- SortedSetDocValues values = reader.getSortedSetDocValues(field.name);
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ SortedSetDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED_SET) {
+ values = docValuesProducer.getSortedSet(fieldInfo);
+ }
+ }
if (values == null) {
values = DocValues.emptySortedSet();
}
toMerge.add(values);
}
- mergeSortedSetField(field, mergeState, toMerge);
+ mergeSortedSetField(mergeFieldInfo, mergeState, toMerge);
} else if (type == DocValuesType.SORTED_NUMERIC) {
List<SortedNumericDocValues> toMerge = new ArrayList<>();
- for (LeafReader reader : mergeState.readers) {
- SortedNumericDocValues values = reader.getSortedNumericDocValues(field.name);
+ for (int i=0;i<mergeState.docValuesProducers.length;i++) {
+ SortedNumericDocValues values = null;
+ DocValuesProducer docValuesProducer = mergeState.docValuesProducers[i];
+ if (docValuesProducer != null) {
+ FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (fieldInfo != null && fieldInfo.getDocValuesType() == DocValuesType.SORTED_NUMERIC) {
+ values = docValuesProducer.getSortedNumeric(fieldInfo);
+ }
+ }
if (values == null) {
- values = DocValues.emptySortedNumeric(reader.maxDoc());
+ values = DocValues.emptySortedNumeric(mergeState.maxDocs[i]);
}
toMerge.add(values);
}
- mergeSortedNumericField(field, mergeState, toMerge);
+ mergeSortedNumericField(mergeFieldInfo, mergeState, toMerge);
} else {
throw new AssertionError("type=" + type);
}
@@ -216,7 +258,7 @@ public abstract class DocValuesConsumer
int docIDUpto;
long nextValue;
boolean nextHasValue;
- LeafReader currentReader;
+ int currentMaxDoc;
NumericDocValues currentValues;
Bits currentLiveDocs;
Bits currentDocsWithField;
@@ -248,13 +290,13 @@ public abstract class DocValuesConsumer
return false;
}
- if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ if (docIDUpto == currentMaxDoc) {
readerUpto++;
if (readerUpto < toMerge.size()) {
- currentReader = mergeState.readers.get(readerUpto);
currentValues = toMerge.get(readerUpto);
- currentLiveDocs = currentReader.getLiveDocs();
currentDocsWithField = docsWithField.get(readerUpto);
+ currentLiveDocs = mergeState.liveDocs[readerUpto];
+ currentMaxDoc = mergeState.maxDocs[readerUpto];
}
docIDUpto = 0;
continue;
@@ -297,7 +339,7 @@ public abstract class DocValuesConsumer
int docIDUpto;
BytesRef nextValue;
BytesRef nextPointer; // points to null if missing, or nextValue
- LeafReader currentReader;
+ int currentMaxDoc;
BinaryDocValues currentValues;
Bits currentLiveDocs;
Bits currentDocsWithField;
@@ -329,13 +371,13 @@ public abstract class DocValuesConsumer
return false;
}
- if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ if (docIDUpto == currentMaxDoc) {
readerUpto++;
if (readerUpto < toMerge.size()) {
- currentReader = mergeState.readers.get(readerUpto);
currentValues = toMerge.get(readerUpto);
currentDocsWithField = docsWithField.get(readerUpto);
- currentLiveDocs = currentReader.getLiveDocs();
+ currentLiveDocs = mergeState.liveDocs[readerUpto];
+ currentMaxDoc = mergeState.maxDocs[readerUpto];
}
docIDUpto = 0;
continue;
@@ -368,8 +410,8 @@ public abstract class DocValuesConsumer
* iterables that filter deleted documents.
*/
public void mergeSortedNumericField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedNumericDocValues> toMerge) throws IOException {
- final LeafReader readers[] = mergeState.readers.toArray(new LeafReader[toMerge.size()]);
- final SortedNumericDocValues dvs[] = toMerge.toArray(new SortedNumericDocValues[toMerge.size()]);
+ final int numReaders = toMerge.size();
+ final SortedNumericDocValues dvs[] = toMerge.toArray(new SortedNumericDocValues[numReaders]);
// step 3: add field
addSortedNumericField(fieldInfo,
@@ -381,7 +423,7 @@ public abstract class DocValuesConsumer
int readerUpto = -1;
int docIDUpto;
int nextValue;
- LeafReader currentReader;
+ int currentMaxDoc;
Bits currentLiveDocs;
boolean nextIsSet;
@@ -407,15 +449,15 @@ public abstract class DocValuesConsumer
private boolean setNext() {
while (true) {
- if (readerUpto == readers.length) {
+ if (readerUpto == numReaders) {
return false;
}
- if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ if (docIDUpto == currentMaxDoc) {
readerUpto++;
- if (readerUpto < readers.length) {
- currentReader = readers[readerUpto];
- currentLiveDocs = currentReader.getLiveDocs();
+ if (readerUpto < numReaders) {
+ currentLiveDocs = mergeState.liveDocs[readerUpto];
+ currentMaxDoc = mergeState.maxDocs[readerUpto];
}
docIDUpto = 0;
continue;
@@ -444,7 +486,7 @@ public abstract class DocValuesConsumer
int readerUpto = -1;
int docIDUpto;
long nextValue;
- LeafReader currentReader;
+ int currentMaxDoc;
Bits currentLiveDocs;
boolean nextIsSet;
int valueUpto;
@@ -472,7 +514,7 @@ public abstract class DocValuesConsumer
private boolean setNext() {
while (true) {
- if (readerUpto == readers.length) {
+ if (readerUpto == numReaders) {
return false;
}
@@ -483,18 +525,18 @@ public abstract class DocValuesConsumer
return true;
}
- if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ if (docIDUpto == currentMaxDoc) {
readerUpto++;
- if (readerUpto < readers.length) {
- currentReader = readers[readerUpto];
- currentLiveDocs = currentReader.getLiveDocs();
+ if (readerUpto < numReaders) {
+ currentLiveDocs = mergeState.liveDocs[readerUpto];
+ currentMaxDoc = mergeState.maxDocs[readerUpto];
}
docIDUpto = 0;
continue;
}
if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto)) {
- assert docIDUpto < currentReader.maxDoc();
+ assert docIDUpto < currentMaxDoc;
SortedNumericDocValues dv = dvs[readerUpto];
dv.setDocument(docIDUpto);
valueUpto = 0;
@@ -519,22 +561,22 @@ public abstract class DocValuesConsumer
* an Iterable that merges ordinals and values and filters deleted documents .
*/
public void mergeSortedField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedDocValues> toMerge) throws IOException {
- final LeafReader readers[] = mergeState.readers.toArray(new LeafReader[toMerge.size()]);
- final SortedDocValues dvs[] = toMerge.toArray(new SortedDocValues[toMerge.size()]);
+ final int numReaders = toMerge.size();
+ final SortedDocValues dvs[] = toMerge.toArray(new SortedDocValues[numReaders]);
// step 1: iterate thru each sub and mark terms still in use
TermsEnum liveTerms[] = new TermsEnum[dvs.length];
long[] weights = new long[liveTerms.length];
- for (int sub = 0; sub < liveTerms.length; sub++) {
- LeafReader reader = readers[sub];
+ for (int sub=0;sub<numReaders;sub++) {
SortedDocValues dv = dvs[sub];
- Bits liveDocs = reader.getLiveDocs();
+ Bits liveDocs = mergeState.liveDocs[sub];
+ int maxDoc = mergeState.maxDocs[sub];
if (liveDocs == null) {
liveTerms[sub] = dv.termsEnum();
weights[sub] = dv.getValueCount();
} else {
LongBitSet bitset = new LongBitSet(dv.getValueCount());
- for (int i = 0; i < reader.maxDoc(); i++) {
+ for (int i = 0; i < maxDoc; i++) {
if (liveDocs.get(i)) {
int ord = dv.getOrd(i);
if (ord >= 0) {
@@ -591,7 +633,7 @@ public abstract class DocValuesConsumer
int readerUpto = -1;
int docIDUpto;
int nextValue;
- LeafReader currentReader;
+ int currentMaxDoc;
Bits currentLiveDocs;
LongValues currentMap;
boolean nextIsSet;
@@ -619,16 +661,16 @@ public abstract class DocValuesConsumer
private boolean setNext() {
while (true) {
- if (readerUpto == readers.length) {
+ if (readerUpto == numReaders) {
return false;
}
- if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ if (docIDUpto == currentMaxDoc) {
readerUpto++;
- if (readerUpto < readers.length) {
- currentReader = readers[readerUpto];
- currentLiveDocs = currentReader.getLiveDocs();
+ if (readerUpto < numReaders) {
currentMap = map.getGlobalOrds(readerUpto);
+ currentLiveDocs = mergeState.liveDocs[readerUpto];
+ currentMaxDoc = mergeState.maxDocs[readerUpto];
}
docIDUpto = 0;
continue;
@@ -658,22 +700,22 @@ public abstract class DocValuesConsumer
* an Iterable that merges ordinals and values and filters deleted documents .
*/
public void mergeSortedSetField(FieldInfo fieldInfo, final MergeState mergeState, List<SortedSetDocValues> toMerge) throws IOException {
- final LeafReader readers[] = mergeState.readers.toArray(new LeafReader[toMerge.size()]);
final SortedSetDocValues dvs[] = toMerge.toArray(new SortedSetDocValues[toMerge.size()]);
-
+ final int numReaders = mergeState.maxDocs.length;
+
// step 1: iterate thru each sub and mark terms still in use
TermsEnum liveTerms[] = new TermsEnum[dvs.length];
long[] weights = new long[liveTerms.length];
for (int sub = 0; sub < liveTerms.length; sub++) {
- LeafReader reader = readers[sub];
SortedSetDocValues dv = dvs[sub];
- Bits liveDocs = reader.getLiveDocs();
+ Bits liveDocs = mergeState.liveDocs[sub];
+ int maxDoc = mergeState.maxDocs[sub];
if (liveDocs == null) {
liveTerms[sub] = dv.termsEnum();
weights[sub] = dv.getValueCount();
} else {
LongBitSet bitset = new LongBitSet(dv.getValueCount());
- for (int i = 0; i < reader.maxDoc(); i++) {
+ for (int i = 0; i < maxDoc; i++) {
if (liveDocs.get(i)) {
dv.setDocument(i);
long ord;
@@ -731,7 +773,7 @@ public abstract class DocValuesConsumer
int readerUpto = -1;
int docIDUpto;
int nextValue;
- LeafReader currentReader;
+ int currentMaxDoc;
Bits currentLiveDocs;
boolean nextIsSet;
@@ -758,15 +800,15 @@ public abstract class DocValuesConsumer
private boolean setNext() {
while (true) {
- if (readerUpto == readers.length) {
+ if (readerUpto == numReaders) {
return false;
}
- if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ if (docIDUpto == currentMaxDoc) {
readerUpto++;
- if (readerUpto < readers.length) {
- currentReader = readers[readerUpto];
- currentLiveDocs = currentReader.getLiveDocs();
+ if (readerUpto < numReaders) {
+ currentLiveDocs = mergeState.liveDocs[readerUpto];
+ currentMaxDoc = mergeState.maxDocs[readerUpto];
}
docIDUpto = 0;
continue;
@@ -798,7 +840,7 @@ public abstract class DocValuesConsumer
int readerUpto = -1;
int docIDUpto;
long nextValue;
- LeafReader currentReader;
+ int currentMaxDoc;
Bits currentLiveDocs;
LongValues currentMap;
boolean nextIsSet;
@@ -829,7 +871,7 @@ public abstract class DocValuesConsumer
private boolean setNext() {
while (true) {
- if (readerUpto == readers.length) {
+ if (readerUpto == numReaders) {
return false;
}
@@ -840,19 +882,19 @@ public abstract class DocValuesConsumer
return true;
}
- if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ if (docIDUpto == currentMaxDoc) {
readerUpto++;
- if (readerUpto < readers.length) {
- currentReader = readers[readerUpto];
- currentLiveDocs = currentReader.getLiveDocs();
+ if (readerUpto < numReaders) {
currentMap = map.getGlobalOrds(readerUpto);
+ currentLiveDocs = mergeState.liveDocs[readerUpto];
+ currentMaxDoc = mergeState.maxDocs[readerUpto];
}
docIDUpto = 0;
continue;
}
if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto)) {
- assert docIDUpto < currentReader.maxDoc();
+ assert docIDUpto < currentMaxDoc;
SortedSetDocValues dv = dvs[readerUpto];
dv.setDocument(docIDUpto);
ordUpto = ordLength = 0;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java Sun Oct 5 14:01:13 2014
@@ -79,4 +79,12 @@ public abstract class DocValuesProducer
* @lucene.internal
*/
public abstract void checkIntegrity() throws IOException;
+
+ /**
+ * Returns an instance optimized for merging.
+ * <p>
+ * The default implementation returns {@code this} */
+ public DocValuesProducer getMergeInstance() throws IOException {
+ return this;
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java Sun Oct 5 14:01:13 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.MappedMultiFields;
import org.apache.lucene.index.MergeState;
@@ -90,11 +89,12 @@ public abstract class FieldsConsumer imp
int docBase = 0;
- for(int readerIndex=0;readerIndex<mergeState.readers.size();readerIndex++) {
- final LeafReader reader = mergeState.readers.get(readerIndex);
- final Fields f = reader.fields();
- final int maxDoc = reader.maxDoc();
+ for(int readerIndex=0;readerIndex<mergeState.fieldsProducers.length;readerIndex++) {
+ final FieldsProducer f = mergeState.fieldsProducers[readerIndex];
+
+ final int maxDoc = mergeState.maxDocs[readerIndex];
if (f != null) {
+ f.checkIntegrity();
slices.add(new ReaderSlice(docBase, maxDoc, readerIndex));
fields.add(f);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java Sun Oct 5 14:01:13 2014
@@ -46,4 +46,12 @@ public abstract class FieldsProducer ext
* @lucene.internal
*/
public abstract void checkIntegrity() throws IOException;
+
+ /**
+ * Returns an instance optimized for merging.
+ * <p>
+ * The default implementation returns {@code this} */
+ public FieldsProducer getMergeInstance() throws IOException {
+ return this;
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FilterCodec.java Sun Oct 5 14:01:13 2014
@@ -99,4 +99,8 @@ public abstract class FilterCodec extend
return delegate.termVectorsFormat();
}
+ @Override
+ public CompoundFormat compoundFormat() {
+ return delegate.compoundFormat();
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/NormsConsumer.java Sun Oct 5 14:01:13 2014
@@ -24,7 +24,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.MergeState;
@@ -72,17 +71,29 @@ public abstract class NormsConsumer impl
* Implementations can override this method
* for more sophisticated merging (bulk-byte copying, etc). */
public void merge(MergeState mergeState) throws IOException {
- for (FieldInfo field : mergeState.fieldInfos) {
- if (field.hasNorms()) {
+ for(NormsProducer normsProducer : mergeState.normsProducers) {
+ if (normsProducer != null) {
+ normsProducer.checkIntegrity();
+ }
+ }
+ for (FieldInfo mergeFieldInfo : mergeState.mergeFieldInfos) {
+ if (mergeFieldInfo.hasNorms()) {
List<NumericDocValues> toMerge = new ArrayList<>();
- for (LeafReader reader : mergeState.readers) {
- NumericDocValues norms = reader.getNormValues(field.name);
+ for (int i=0;i<mergeState.normsProducers.length;i++) {
+ NormsProducer normsProducer = mergeState.normsProducers[i];
+ NumericDocValues norms = null;
+ if (normsProducer != null) {
+ FieldInfo fieldInfo = mergeState.fieldInfos[i].fieldInfo(mergeFieldInfo.name);
+ if (fieldInfo != null && fieldInfo.hasNorms()) {
+ norms = normsProducer.getNorms(fieldInfo);
+ }
+ }
if (norms == null) {
norms = DocValues.emptyNumeric();
}
toMerge.add(norms);
}
- mergeNormsField(field, mergeState, toMerge);
+ mergeNormsField(mergeFieldInfo, mergeState, toMerge);
}
}
}
@@ -104,7 +115,7 @@ public abstract class NormsConsumer impl
int readerUpto = -1;
int docIDUpto;
long nextValue;
- LeafReader currentReader;
+ int maxDoc;
NumericDocValues currentValues;
Bits currentLiveDocs;
boolean nextIsSet;
@@ -135,12 +146,12 @@ public abstract class NormsConsumer impl
return false;
}
- if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ if (currentValues == null || docIDUpto == maxDoc) {
readerUpto++;
if (readerUpto < toMerge.size()) {
- currentReader = mergeState.readers.get(readerUpto);
currentValues = toMerge.get(readerUpto);
- currentLiveDocs = currentReader.getLiveDocs();
+ currentLiveDocs = mergeState.liveDocs[readerUpto];
+ maxDoc = mergeState.maxDocs[readerUpto];
}
docIDUpto = 0;
continue;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/NormsProducer.java Sun Oct 5 14:01:13 2014
@@ -47,4 +47,12 @@ public abstract class NormsProducer impl
* @lucene.internal
*/
public abstract void checkIntegrity() throws IOException;
+
+ /**
+ * Returns an instance optimized for merging.
+ * <p>
+ * The default implementation returns {@code this} */
+ public NormsProducer getMergeInstance() throws IOException {
+ return this;
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java Sun Oct 5 14:01:13 2014
@@ -36,8 +36,8 @@ public abstract class StoredFieldsReader
protected StoredFieldsReader() {
}
- /** Visit the stored fields for document <code>n</code> */
- public abstract void visitDocument(int n, StoredFieldVisitor visitor) throws IOException;
+ /** Visit the stored fields for document <code>docID</code> */
+ public abstract void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException;
@Override
public abstract StoredFieldsReader clone();
@@ -50,4 +50,12 @@ public abstract class StoredFieldsReader
* @lucene.internal
*/
public abstract void checkIntegrity() throws IOException;
+
+ /**
+ * Returns an instance optimized for merging.
+ * <p>
+ * The default implementation returns {@code this} */
+ public StoredFieldsReader getMergeInstance() throws IOException {
+ return this;
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java Sun Oct 5 14:01:13 2014
@@ -19,13 +19,13 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.index.LeafReader;
/**
* Codec API for writing stored fields:
@@ -82,11 +82,13 @@ public abstract class StoredFieldsWriter
* merging (bulk-byte copying, etc). */
public int merge(MergeState mergeState) throws IOException {
int docCount = 0;
- for (LeafReader reader : mergeState.readers) {
- final int maxDoc = reader.maxDoc();
- final Bits liveDocs = reader.getLiveDocs();
- for (int i = 0; i < maxDoc; i++) {
- if (liveDocs != null && !liveDocs.get(i)) {
+ for (int i=0;i<mergeState.storedFieldsReaders.length;i++) {
+ StoredFieldsReader storedFieldsReader = mergeState.storedFieldsReaders[i];
+ storedFieldsReader.checkIntegrity();
+ int maxDoc = mergeState.maxDocs[i];
+ Bits liveDocs = mergeState.liveDocs[i];
+ for (int docID=0;docID<maxDoc;docID++) {
+ if (liveDocs != null && !liveDocs.get(docID)) {
// skip deleted docs
continue;
}
@@ -96,13 +98,15 @@ public abstract class StoredFieldsWriter
// on the fly?
// NOTE: it's very important to first assign to doc then pass it to
// fieldsWriter.addDocument; see LUCENE-1282
- StoredDocument doc = reader.document(i);
- addDocument(doc, mergeState.fieldInfos);
+ DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
+ storedFieldsReader.visitDocument(docID, visitor);
+ StoredDocument doc = visitor.getDocument();
+ addDocument(doc, mergeState.mergeFieldInfos);
docCount++;
mergeState.checkAbort.work(300);
}
}
- finish(mergeState.fieldInfos, docCount);
+ finish(mergeState.mergeFieldInfos, docCount);
return docCount;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java Sun Oct 5 14:01:13 2014
@@ -56,4 +56,12 @@ public abstract class TermVectorsReader
* read term vectors. */
@Override
public abstract TermVectorsReader clone();
+
+ /**
+ * Returns an instance optimized for merging.
+ * <p>
+ * The default implementation returns {@code this} */
+ public TermVectorsReader getMergeInstance() throws IOException {
+ return this;
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java Sun Oct 5 14:01:13 2014
@@ -21,7 +21,6 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.Iterator;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@@ -177,25 +176,34 @@ public abstract class TermVectorsWriter
* merging (bulk-byte copying, etc). */
public int merge(MergeState mergeState) throws IOException {
int docCount = 0;
- for (int i = 0; i < mergeState.readers.size(); i++) {
- final LeafReader reader = mergeState.readers.get(i);
- final int maxDoc = reader.maxDoc();
- final Bits liveDocs = reader.getLiveDocs();
+ int numReaders = mergeState.maxDocs.length;
+ for (int i = 0; i < numReaders; i++) {
+ int maxDoc = mergeState.maxDocs[i];
+ Bits liveDocs = mergeState.liveDocs[i];
+ TermVectorsReader termVectorsReader = mergeState.termVectorsReaders[i];
+ if (termVectorsReader != null) {
+ termVectorsReader.checkIntegrity();
+ }
- for (int docID = 0; docID < maxDoc; docID++) {
+ for (int docID=0;docID<maxDoc;docID++) {
if (liveDocs != null && !liveDocs.get(docID)) {
// skip deleted docs
continue;
}
// NOTE: it's very important to first assign to vectors then pass it to
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
- Fields vectors = reader.getTermVectors(docID);
+ Fields vectors;
+ if (termVectorsReader == null) {
+ vectors = null;
+ } else {
+ vectors = termVectorsReader.get(docID);
+ }
addAllDocVectors(vectors, mergeState);
docCount++;
mergeState.checkAbort.work(300);
}
}
- finish(mergeState.fieldInfos, docCount);
+ finish(mergeState.mergeFieldInfos, docCount);
return docCount;
}
@@ -227,7 +235,7 @@ public abstract class TermVectorsWriter
int fieldCount = 0;
for(String fieldName : vectors) {
fieldCount++;
- final FieldInfo fieldInfo = mergeState.fieldInfos.fieldInfo(fieldName);
+ final FieldInfo fieldInfo = mergeState.mergeFieldInfos.fieldInfo(fieldName);
assert lastFieldName == null || fieldName.compareTo(lastFieldName) > 0: "lastFieldName=" + lastFieldName + " fieldName=" + fieldName;
lastFieldName = fieldName;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java Sun Oct 5 14:01:13 2014
@@ -64,7 +64,7 @@ public class CompressingStoredFieldsForm
* <p>
* <code>formatName</code> is the name of the format. This name will be used
* in the file formats to perform
- * {@link CodecUtil#checkHeader(org.apache.lucene.store.DataInput, String, int, int) codec header checks}.
+ * {@link CodecUtil#checkSegmentHeader codec header checks}.
* <p>
* <code>segmentSuffix</code> is the segment suffix. This suffix is added to
* the result file name only if it's not the empty string.
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java Sun Oct 5 14:01:13 2014
@@ -27,8 +27,6 @@ import static org.apache.lucene.codecs.c
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.STRING;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.TYPE_BITS;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.TYPE_MASK;
-import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.VERSION_BIG_CHUNKS;
-import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.VERSION_CHECKSUM;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.VERSION_CURRENT;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.VERSION_START;
import static org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.FIELDS_EXTENSION;
@@ -109,66 +107,61 @@ public final class CompressingStoredFiel
boolean success = false;
fieldInfos = fn;
numDocs = si.getDocCount();
- ChecksumIndexInput indexStream = null;
- try {
- final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION);
- final String fieldsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION);
- // Load the index into memory
- indexStream = d.openChecksumInput(indexStreamFN, context);
- final String codecNameIdx = formatName + CODEC_SFX_IDX;
- version = CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT);
- assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
- indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
-
- long maxPointer = -1;
-
- if (version >= VERSION_CHECKSUM) {
+
+ int version = -1;
+ long maxPointer = -1;
+ CompressingStoredFieldsIndexReader indexReader = null;
+
+ // Load the index into memory
+ final String indexName = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_INDEX_EXTENSION);
+ try (ChecksumIndexInput indexStream = d.openChecksumInput(indexName, context)) {
+ Throwable priorE = null;
+ try {
+ final String codecNameIdx = formatName + CODEC_SFX_IDX;
+ version = CodecUtil.checkSegmentHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
+ assert CodecUtil.segmentHeaderLength(codecNameIdx, segmentSuffix) == indexStream.getFilePointer();
+ indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
maxPointer = indexStream.readVLong();
- CodecUtil.checkFooter(indexStream);
- } else {
- CodecUtil.checkEOF(indexStream);
+ } catch (Throwable exception) {
+ priorE = exception;
+ } finally {
+ CodecUtil.checkFooter(indexStream, priorE);
}
- indexStream.close();
- indexStream = null;
+ }
+
+ this.version = version;
+ this.maxPointer = maxPointer;
+ this.indexReader = indexReader;
+ final String fieldsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, FIELDS_EXTENSION);
+ try {
// Open the data file and read metadata
fieldsStream = d.openInput(fieldsStreamFN, context);
- if (version >= VERSION_CHECKSUM) {
- if (maxPointer + CodecUtil.footerLength() != fieldsStream.length()) {
- throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + fieldsStream.length(), fieldsStream);
- }
- } else {
- maxPointer = fieldsStream.length();
+ if (maxPointer + CodecUtil.footerLength() != fieldsStream.length()) {
+ throw new CorruptIndexException("Invalid fieldsStream maxPointer (file truncated?): maxPointer=" + maxPointer + ", length=" + fieldsStream.length(), fieldsStream);
}
- this.maxPointer = maxPointer;
final String codecNameDat = formatName + CODEC_SFX_DAT;
- final int fieldsVersion = CodecUtil.checkHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT);
+ final int fieldsVersion = CodecUtil.checkSegmentHeader(fieldsStream, codecNameDat, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
if (version != fieldsVersion) {
throw new CorruptIndexException("Version mismatch between stored fields index and data: " + version + " != " + fieldsVersion, fieldsStream);
}
- assert CodecUtil.headerLength(codecNameDat) == fieldsStream.getFilePointer();
+ assert CodecUtil.segmentHeaderLength(codecNameDat, segmentSuffix) == fieldsStream.getFilePointer();
- if (version >= VERSION_BIG_CHUNKS) {
- chunkSize = fieldsStream.readVInt();
- } else {
- chunkSize = -1;
- }
+ chunkSize = fieldsStream.readVInt();
packedIntsVersion = fieldsStream.readVInt();
decompressor = compressionMode.newDecompressor();
this.bytes = new BytesRef();
- if (version >= VERSION_CHECKSUM) {
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(fieldsStream);
- }
+ // NOTE: data file is too costly to verify checksum against all the bytes on open,
+ // but for now we at least verify proper structure of the checksum footer: which looks
+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+ // such as file truncation.
+ CodecUtil.retrieveChecksum(fieldsStream);
success = true;
} finally {
if (!success) {
- IOUtils.closeWhileHandlingException(this, indexStream);
+ IOUtils.closeWhileHandlingException(this);
}
}
}
@@ -310,7 +303,7 @@ public final class CompressingStoredFiel
}
final DataInput documentInput;
- if (version >= VERSION_BIG_CHUNKS && totalLength >= 2 * chunkSize) {
+ if (totalLength >= 2 * chunkSize) {
assert chunkSize > 0;
assert offset < chunkSize;
@@ -497,7 +490,7 @@ public final class CompressingStoredFiel
void decompress() throws IOException {
// decompress data
final int chunkSize = chunkSize();
- if (version >= VERSION_BIG_CHUNKS && chunkSize >= 2 * CompressingStoredFieldsReader.this.chunkSize) {
+ if (chunkSize >= 2 * CompressingStoredFieldsReader.this.chunkSize) {
bytes.offset = bytes.length = 0;
for (int decompressed = 0; decompressed < chunkSize; ) {
final int toDecompress = Math.min(chunkSize - decompressed, CompressingStoredFieldsReader.this.chunkSize);
@@ -519,10 +512,8 @@ public final class CompressingStoredFiel
* Check integrity of the data. The iterator is not usable after this method has been called.
*/
void checkIntegrity() throws IOException {
- if (version >= VERSION_CHECKSUM) {
- fieldsStream.seek(fieldsStream.length() - CodecUtil.footerLength());
- CodecUtil.checkFooter(fieldsStream);
- }
+ fieldsStream.seek(fieldsStream.length() - CodecUtil.footerLength());
+ CodecUtil.checkFooter(fieldsStream);
}
}
@@ -539,9 +530,7 @@ public final class CompressingStoredFiel
@Override
public void checkIntegrity() throws IOException {
- if (version >= VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(fieldsStream);
- }
+ CodecUtil.checksumEntireFile(fieldsStream);
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Sun Oct 5 14:01:13 2014
@@ -24,11 +24,13 @@ import org.apache.lucene.codecs.CodecUti
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.compressing.CompressingStoredFieldsReader.ChunkIterator;
-import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReader;
@@ -73,9 +75,7 @@ public final class CompressingStoredFiel
static final String CODEC_SFX_IDX = "Index";
static final String CODEC_SFX_DAT = "Data";
static final int VERSION_START = 0;
- static final int VERSION_BIG_CHUNKS = 1;
- static final int VERSION_CHECKSUM = 2;
- static final int VERSION_CURRENT = VERSION_CHECKSUM;
+ static final int VERSION_CURRENT = VERSION_START;
private final Directory directory;
private final String segment;
@@ -118,10 +118,10 @@ public final class CompressingStoredFiel
final String codecNameIdx = formatName + CODEC_SFX_IDX;
final String codecNameDat = formatName + CODEC_SFX_DAT;
- CodecUtil.writeHeader(indexStream, codecNameIdx, VERSION_CURRENT);
- CodecUtil.writeHeader(fieldsStream, codecNameDat, VERSION_CURRENT);
- assert CodecUtil.headerLength(codecNameDat) == fieldsStream.getFilePointer();
- assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
+ CodecUtil.writeSegmentHeader(indexStream, codecNameIdx, VERSION_CURRENT, si.getId(), segmentSuffix);
+ CodecUtil.writeSegmentHeader(fieldsStream, codecNameDat, VERSION_CURRENT, si.getId(), segmentSuffix);
+ assert CodecUtil.segmentHeaderLength(codecNameDat, segmentSuffix) == fieldsStream.getFilePointer();
+ assert CodecUtil.segmentHeaderLength(codecNameIdx, segmentSuffix) == indexStream.getFilePointer();
indexWriter = new CompressingStoredFieldsIndexWriter(indexStream);
indexStream = null;
@@ -333,32 +333,36 @@ public final class CompressingStoredFiel
@Override
public int merge(MergeState mergeState) throws IOException {
int docCount = 0;
- int idx = 0;
+ int numReaders = mergeState.maxDocs.length;
MatchingReaders matching = new MatchingReaders(mergeState);
- for (LeafReader reader : mergeState.readers) {
- final SegmentReader matchingSegmentReader = matching.matchingSegmentReaders[idx++];
+ for (int readerIndex=0;readerIndex<numReaders;readerIndex++) {
CompressingStoredFieldsReader matchingFieldsReader = null;
- if (matchingSegmentReader != null) {
- final StoredFieldsReader fieldsReader = matchingSegmentReader.getFieldsReader();
+ if (matching.matchingReaders[readerIndex]) {
+ final StoredFieldsReader fieldsReader = mergeState.storedFieldsReaders[readerIndex];
// we can only bulk-copy if the matching reader is also a CompressingStoredFieldsReader
if (fieldsReader != null && fieldsReader instanceof CompressingStoredFieldsReader) {
matchingFieldsReader = (CompressingStoredFieldsReader) fieldsReader;
}
}
- final int maxDoc = reader.maxDoc();
- final Bits liveDocs = reader.getLiveDocs();
+ final int maxDoc = mergeState.maxDocs[readerIndex];
+ final Bits liveDocs = mergeState.liveDocs[readerIndex];
if (matchingFieldsReader == null
|| matchingFieldsReader.getVersion() != VERSION_CURRENT // means reader version is not the same as the writer version
|| matchingFieldsReader.getCompressionMode() != compressionMode
|| matchingFieldsReader.getChunkSize() != chunkSize) { // the way data is decompressed depends on the chunk size
// naive merge...
+ StoredFieldsReader storedFieldsReader = mergeState.storedFieldsReaders[readerIndex];
+ if (storedFieldsReader != null) {
+ storedFieldsReader.checkIntegrity();
+ }
for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) {
- StoredDocument doc = reader.document(i);
- addDocument(doc, mergeState.fieldInfos);
+ DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
+ storedFieldsReader.visitDocument(i, visitor);
+ addDocument(visitor.getDocument(), mergeState.mergeFieldInfos);
++docCount;
mergeState.checkAbort.work(300);
}
@@ -400,7 +404,7 @@ public final class CompressingStoredFiel
}
}
}
- finish(mergeState.fieldInfos, docCount);
+ finish(mergeState.mergeFieldInfos, docCount);
return docCount;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsFormat.java Sun Oct 5 14:01:13 2014
@@ -46,7 +46,7 @@ public class CompressingTermVectorsForma
* <p>
* <code>formatName</code> is the name of the format. This name will be used
* in the file formats to perform
- * {@link CodecUtil#checkHeader(org.apache.lucene.store.DataInput, String, int, int) codec header checks}.
+ * {@link CodecUtil#checkSegmentHeader codec header checks}.
* <p>
* The <code>compressionMode</code> parameter allows you to choose between
* compression algorithms that have various compression and decompression
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java Sun Oct 5 14:01:13 2014
@@ -28,7 +28,6 @@ import static org.apache.lucene.codecs.c
import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION;
import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_CURRENT;
import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_START;
-import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_CHECKSUM;
import java.io.Closeable;
import java.io.IOException;
@@ -106,44 +105,47 @@ public final class CompressingTermVector
boolean success = false;
fieldInfos = fn;
numDocs = si.getDocCount();
- ChecksumIndexInput indexStream = null;
- try {
- // Load the index into memory
- final String indexStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION);
- indexStream = d.openChecksumInput(indexStreamFN, context);
- final String codecNameIdx = formatName + CODEC_SFX_IDX;
- version = CodecUtil.checkHeader(indexStream, codecNameIdx, VERSION_START, VERSION_CURRENT);
- assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
- indexReader = new CompressingStoredFieldsIndexReader(indexStream, si);
-
- if (version >= VERSION_CHECKSUM) {
- indexStream.readVLong(); // the end of the data file
- CodecUtil.checkFooter(indexStream);
- } else {
- CodecUtil.checkEOF(indexStream);
- }
- indexStream.close();
- indexStream = null;
+ int version = -1;
+ CompressingStoredFieldsIndexReader indexReader = null;
+
+ // Load the index into memory
+ final String indexName = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_INDEX_EXTENSION);
+ try (ChecksumIndexInput input = d.openChecksumInput(indexName, context)) {
+ Throwable priorE = null;
+ try {
+ final String codecNameIdx = formatName + CODEC_SFX_IDX;
+ version = CodecUtil.checkSegmentHeader(input, codecNameIdx, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
+ assert CodecUtil.segmentHeaderLength(codecNameIdx, segmentSuffix) == input.getFilePointer();
+ indexReader = new CompressingStoredFieldsIndexReader(input, si);
+ input.readVLong(); // the end of the data file
+ } catch (Throwable exception) {
+ priorE = exception;
+ } finally {
+ CodecUtil.checkFooter(input, priorE);
+ }
+ }
+
+ this.version = version;
+ this.indexReader = indexReader;
+ try {
// Open the data file and read metadata
final String vectorsStreamFN = IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);
vectorsStream = d.openInput(vectorsStreamFN, context);
final String codecNameDat = formatName + CODEC_SFX_DAT;
- int version2 = CodecUtil.checkHeader(vectorsStream, codecNameDat, VERSION_START, VERSION_CURRENT);
+ int version2 = CodecUtil.checkSegmentHeader(vectorsStream, codecNameDat, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
if (version != version2) {
throw new CorruptIndexException("Version mismatch between stored fields index and data: " + version + " != " + version2, vectorsStream);
}
- assert CodecUtil.headerLength(codecNameDat) == vectorsStream.getFilePointer();
+ assert CodecUtil.segmentHeaderLength(codecNameDat, segmentSuffix) == vectorsStream.getFilePointer();
long pos = vectorsStream.getFilePointer();
- if (version >= VERSION_CHECKSUM) {
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(vectorsStream);
- vectorsStream.seek(pos);
- }
+ // NOTE: data file is too costly to verify checksum against all the bytes on open,
+ // but for now we at least verify proper structure of the checksum footer: which looks
+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+ // such as file truncation.
+ CodecUtil.retrieveChecksum(vectorsStream);
+ vectorsStream.seek(pos);
packedIntsVersion = vectorsStream.readVInt();
chunkSize = vectorsStream.readVInt();
@@ -153,7 +155,7 @@ public final class CompressingTermVector
success = true;
} finally {
if (!success) {
- IOUtils.closeWhileHandlingException(this, indexStream);
+ IOUtils.closeWhileHandlingException(this);
}
}
}
@@ -1078,9 +1080,7 @@ public final class CompressingTermVector
@Override
public void checkIntegrity() throws IOException {
- if (version >= VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(vectorsStream);
- }
+ CodecUtil.checksumEntireFile(vectorsStream);
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java Sun Oct 5 14:01:13 2014
@@ -28,11 +28,12 @@ import java.util.TreeSet;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.TermVectorsWriter;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentReader;
@@ -68,8 +69,7 @@ public final class CompressingTermVector
static final String CODEC_SFX_DAT = "Data";
static final int VERSION_START = 0;
- static final int VERSION_CHECKSUM = 1;
- static final int VERSION_CURRENT = VERSION_CHECKSUM;
+ static final int VERSION_CURRENT = VERSION_START;
static final int BLOCK_SIZE = 64;
@@ -231,10 +231,10 @@ public final class CompressingTermVector
final String codecNameIdx = formatName + CODEC_SFX_IDX;
final String codecNameDat = formatName + CODEC_SFX_DAT;
- CodecUtil.writeHeader(indexStream, codecNameIdx, VERSION_CURRENT);
- CodecUtil.writeHeader(vectorsStream, codecNameDat, VERSION_CURRENT);
- assert CodecUtil.headerLength(codecNameDat) == vectorsStream.getFilePointer();
- assert CodecUtil.headerLength(codecNameIdx) == indexStream.getFilePointer();
+ CodecUtil.writeSegmentHeader(indexStream, codecNameIdx, VERSION_CURRENT, si.getId(), segmentSuffix);
+ CodecUtil.writeSegmentHeader(vectorsStream, codecNameDat, VERSION_CURRENT, si.getId(), segmentSuffix);
+ assert CodecUtil.segmentHeaderLength(codecNameDat, segmentSuffix) == vectorsStream.getFilePointer();
+ assert CodecUtil.segmentHeaderLength(codecNameIdx, segmentSuffix) == indexStream.getFilePointer();
indexWriter = new CompressingStoredFieldsIndexWriter(indexStream);
indexStream = null;
@@ -728,23 +728,22 @@ public final class CompressingTermVector
@Override
public int merge(MergeState mergeState) throws IOException {
int docCount = 0;
- int idx = 0;
+ int numReaders = mergeState.maxDocs.length;
MatchingReaders matching = new MatchingReaders(mergeState);
- for (LeafReader reader : mergeState.readers) {
- final SegmentReader matchingSegmentReader = matching.matchingSegmentReaders[idx++];
+ for (int readerIndex=0;readerIndex<numReaders;readerIndex++) {
CompressingTermVectorsReader matchingVectorsReader = null;
- if (matchingSegmentReader != null) {
- final TermVectorsReader vectorsReader = matchingSegmentReader.getTermVectorsReader();
+ final TermVectorsReader vectorsReader = mergeState.termVectorsReaders[readerIndex];
+ if (matching.matchingReaders[readerIndex]) {
// we can only bulk-copy if the matching reader is also a CompressingTermVectorsReader
if (vectorsReader != null && vectorsReader instanceof CompressingTermVectorsReader) {
matchingVectorsReader = (CompressingTermVectorsReader) vectorsReader;
}
}
- final int maxDoc = reader.maxDoc();
- final Bits liveDocs = reader.getLiveDocs();
+ final int maxDoc = mergeState.maxDocs[readerIndex];
+ final Bits liveDocs = mergeState.liveDocs[readerIndex];
if (matchingVectorsReader == null
|| matchingVectorsReader.getVersion() != VERSION_CURRENT
@@ -752,8 +751,16 @@ public final class CompressingTermVector
|| matchingVectorsReader.getChunkSize() != chunkSize
|| matchingVectorsReader.getPackedIntsVersion() != PackedInts.VERSION_CURRENT) {
// naive merge...
+ if (vectorsReader != null) {
+ vectorsReader.checkIntegrity();
+ }
for (int i = nextLiveDoc(0, liveDocs, maxDoc); i < maxDoc; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) {
- final Fields vectors = reader.getTermVectors(i);
+ Fields vectors;
+ if (vectorsReader == null) {
+ vectors = null;
+ } else {
+ vectors = vectorsReader.get(i);
+ }
addAllDocVectors(vectors, mergeState);
++docCount;
mergeState.checkAbort.work(300);
@@ -775,8 +782,8 @@ public final class CompressingTermVector
&& (i == 0 || index.getStartPointer(i - 1) < startPointer)) { // start of a chunk
final int docBase = vectorsStream.readVInt();
final int chunkDocs = vectorsStream.readVInt();
- assert docBase + chunkDocs <= matchingSegmentReader.maxDoc();
- if (docBase + chunkDocs < matchingSegmentReader.maxDoc()
+ assert docBase + chunkDocs <= maxDoc;
+ if (docBase + chunkDocs < maxDoc
&& nextDeletedDoc(docBase, liveDocs, docBase + chunkDocs) == docBase + chunkDocs) {
final long chunkEnd = index.getStartPointer(docBase + chunkDocs);
final long chunkLength = chunkEnd - vectorsStream.getFilePointer();
@@ -790,14 +797,24 @@ public final class CompressingTermVector
i = nextLiveDoc(docBase + chunkDocs, liveDocs, maxDoc);
} else {
for (; i < docBase + chunkDocs; i = nextLiveDoc(i + 1, liveDocs, maxDoc)) {
- final Fields vectors = reader.getTermVectors(i);
+ Fields vectors;
+ if (vectorsReader == null) {
+ vectors = null;
+ } else {
+ vectors = vectorsReader.get(i);
+ }
addAllDocVectors(vectors, mergeState);
++docCount;
mergeState.checkAbort.work(300);
}
}
} else {
- final Fields vectors = reader.getTermVectors(i);
+ Fields vectors;
+ if (vectorsReader == null) {
+ vectors = null;
+ } else {
+ vectors = vectorsReader.get(i);
+ }
addAllDocVectors(vectors, mergeState);
++docCount;
mergeState.checkAbort.work(300);
@@ -809,7 +826,7 @@ public final class CompressingTermVector
CodecUtil.checkFooter(vectorsStream);
}
}
- finish(mergeState.fieldInfos, docCount);
+ finish(mergeState.mergeFieldInfos, docCount);
return docCount;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/MatchingReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/MatchingReaders.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/MatchingReaders.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/MatchingReaders.java Sun Oct 5 14:01:13 2014
@@ -17,9 +17,7 @@ package org.apache.lucene.codecs.compres
* limitations under the License.
*/
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentReader;
@@ -32,54 +30,42 @@ class MatchingReaders {
/** {@link SegmentReader}s that have identical field
* name/number mapping, so their stored fields and term
* vectors may be bulk merged. */
- final SegmentReader[] matchingSegmentReaders;
+ final boolean[] matchingReaders;
- /** How many {@link #matchingSegmentReaders} are set. */
+ /** How many {@link #matchingReaders} are set. */
final int count;
MatchingReaders(MergeState mergeState) {
// If the i'th reader is a SegmentReader and has
// identical fieldName -> number mapping, then this
// array will be non-null at position i:
- int numReaders = mergeState.readers.size();
+ int numReaders = mergeState.maxDocs.length;
int matchedCount = 0;
- matchingSegmentReaders = new SegmentReader[numReaders];
+ matchingReaders = new boolean[numReaders];
// If this reader is a SegmentReader, and all of its
// field name -> number mappings match the "merged"
// FieldInfos, then we can do a bulk copy of the
// stored fields:
+
+ nextReader:
for (int i = 0; i < numReaders; i++) {
- LeafReader reader = mergeState.readers.get(i);
- // TODO: we may be able to broaden this to
- // non-SegmentReaders, since FieldInfos is now
- // required? But... this'd also require exposing
- // bulk-copy (TVs and stored fields) API in foreign
- // readers..
- if (reader instanceof SegmentReader) {
- SegmentReader segmentReader = (SegmentReader) reader;
- boolean same = true;
- FieldInfos segmentFieldInfos = segmentReader.getFieldInfos();
- for (FieldInfo fi : segmentFieldInfos) {
- FieldInfo other = mergeState.fieldInfos.fieldInfo(fi.number);
- if (other == null || !other.name.equals(fi.name)) {
- same = false;
- break;
- }
- }
- if (same) {
- matchingSegmentReaders[i] = segmentReader;
- matchedCount++;
+ for (FieldInfo fi : mergeState.fieldInfos[i]) {
+ FieldInfo other = mergeState.mergeFieldInfos.fieldInfo(fi.number);
+ if (other == null || !other.name.equals(fi.name)) {
+ continue nextReader;
}
}
+ matchingReaders[i] = true;
+ matchedCount++;
}
this.count = matchedCount;
if (mergeState.infoStream.isEnabled("SM")) {
- mergeState.infoStream.message("SM", "merge store matchedCount=" + count + " vs " + mergeState.readers.size());
- if (count != mergeState.readers.size()) {
- mergeState.infoStream.message("SM", "" + (mergeState.readers.size() - count) + " non-bulk merges");
+ mergeState.infoStream.message("SM", "merge store matchedCount=" + count + " vs " + numReaders);
+ if (count != numReaders) {
+ mergeState.infoStream.message("SM", "" + (numReaders - count) + " non-bulk merges");
}
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene410/Lucene410DocValuesProducer.java Sun Oct 5 14:01:13 2014
@@ -92,10 +92,33 @@ class Lucene410DocValuesProducer extends
private final Map<String,MonotonicBlockPackedReader> ordIndexInstances = new HashMap<>();
private final Map<String,ReverseTermsIndex> reverseIndexInstances = new HashMap<>();
+ private final boolean merging;
+
+ // clone for merge: when merging we don't do any instances.put()s
+ Lucene410DocValuesProducer(Lucene410DocValuesProducer original) throws IOException {
+ assert Thread.holdsLock(original);
+ numerics.putAll(original.numerics);
+ binaries.putAll(original.binaries);
+ sortedSets.putAll(original.sortedSets);
+ sortedNumerics.putAll(original.sortedNumerics);
+ ords.putAll(original.ords);
+ ordIndexes.putAll(original.ordIndexes);
+ numFields = original.numFields;
+ ramBytesUsed = new AtomicLong(original.ramBytesUsed.get());
+ data = original.data.clone();
+ maxDoc = original.maxDoc;
+
+ addressInstances.putAll(original.addressInstances);
+ ordIndexInstances.putAll(original.ordIndexInstances);
+ reverseIndexInstances.putAll(original.reverseIndexInstances);
+ merging = true;
+ }
+
/** expert: instantiates a new reader */
Lucene410DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
this.maxDoc = state.segmentInfo.getDocCount();
+ merging = false;
int version = -1;
int numFields = -1;
@@ -441,8 +464,10 @@ class Lucene410DocValuesProducer extends
if (addresses == null) {
data.seek(bytes.addressesOffset);
addresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, bytes.count+1, false);
- addressInstances.put(field.name, addresses);
- ramBytesUsed.addAndGet(addresses.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ if (!merging) {
+ addressInstances.put(field.name, addresses);
+ ramBytesUsed.addAndGet(addresses.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
}
return addresses;
}
@@ -479,8 +504,10 @@ class Lucene410DocValuesProducer extends
data.seek(bytes.addressesOffset);
final long size = (bytes.count + INTERVAL_MASK) >>> INTERVAL_SHIFT;
addresses = MonotonicBlockPackedReader.of(data, bytes.packedIntsVersion, bytes.blockSize, size, false);
- addressInstances.put(field.name, addresses);
- ramBytesUsed.addAndGet(addresses.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ if (!merging) {
+ addressInstances.put(field.name, addresses);
+ ramBytesUsed.addAndGet(addresses.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
}
return addresses;
}
@@ -497,8 +524,10 @@ class Lucene410DocValuesProducer extends
PagedBytes pagedBytes = new PagedBytes(15);
pagedBytes.copy(data, dataSize);
index.terms = pagedBytes.freeze(true);
- reverseIndexInstances.put(field.name, index);
- ramBytesUsed.addAndGet(index.ramBytesUsed());
+ if (!merging) {
+ reverseIndexInstances.put(field.name, index);
+ ramBytesUsed.addAndGet(index.ramBytesUsed());
+ }
}
return index;
}
@@ -560,8 +589,10 @@ class Lucene410DocValuesProducer extends
if (instance == null) {
data.seek(entry.offset);
instance = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, entry.count+1, false);
- ordIndexInstances.put(field.name, instance);
- ramBytesUsed.addAndGet(instance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ if (!merging) {
+ ordIndexInstances.put(field.name, instance);
+ ramBytesUsed.addAndGet(instance.ramBytesUsed() + RamUsageEstimator.NUM_BYTES_INT);
+ }
}
return instance;
}
@@ -726,6 +757,11 @@ class Lucene410DocValuesProducer extends
}
@Override
+ public synchronized DocValuesProducer getMergeInstance() throws IOException {
+ return new Lucene410DocValuesProducer(this);
+ }
+
+ @Override
public void close() throws IOException {
data.close();
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50Codec.java Sun Oct 5 14:01:13 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.codecs.lucene5
*/
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FilterCodec;
@@ -27,9 +28,6 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat;
-import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat;
-import org.apache.lucene.codecs.lucene49.Lucene49NormsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -44,11 +42,12 @@ import org.apache.lucene.codecs.perfield
* @lucene.experimental
*/
public class Lucene50Codec extends Codec {
- private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
- private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
+ private final StoredFieldsFormat fieldsFormat = new Lucene50StoredFieldsFormat();
+ private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = new Lucene50FieldInfosFormat();
private final SegmentInfoFormat segmentInfosFormat = new Lucene50SegmentInfoFormat();
private final LiveDocsFormat liveDocsFormat = new Lucene50LiveDocsFormat();
+ private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
@@ -99,6 +98,11 @@ public class Lucene50Codec extends Codec
return liveDocsFormat;
}
+ @Override
+ public final CompoundFormat compoundFormat() {
+ return compoundFormat;
+ }
+
/** Returns the postings format that should be used for writing
* new segments of <code>field</code>.
*
@@ -125,7 +129,7 @@ public class Lucene50Codec extends Codec
private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene41");
private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene410");
- private final NormsFormat normsFormat = new Lucene49NormsFormat();
+ private final NormsFormat normsFormat = new Lucene50NormsFormat();
@Override
public final NormsFormat normsFormat() {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java Sun Oct 5 14:01:13 2014
@@ -25,7 +25,6 @@ import org.apache.lucene.codecs.FieldInf
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.DataOutput;
/**
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java?rev=1629499&r1=1629498&r2=1629499&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosReader.java Sun Oct 5 14:01:13 2014
@@ -57,7 +57,7 @@ final class Lucene50FieldInfosReader ext
CodecUtil.checkSegmentHeader(input, Lucene50FieldInfosFormat.CODEC_NAME,
Lucene50FieldInfosFormat.FORMAT_START,
Lucene50FieldInfosFormat.FORMAT_CURRENT,
- segmentInfo.getId());
+ segmentInfo.getId(), segmentSuffix);
final int size = input.readVInt(); //read in the size
infos = new FieldInfo[size];