You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2014/04/04 12:27:14 UTC
svn commit: r1584603 [5/12] - in /lucene/dev/branches/solr5914: ./
dev-tools/ dev-tools/idea/solr/core/src/test/ lucene/ lucene/analysis/
lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/br/
lucene/analysis/common/src/...
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java Fri Apr 4 10:27:05 2014
@@ -105,9 +105,7 @@ final class Lucene40DocValuesReader exte
default:
throw new AssertionError();
}
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
- }
+ CodecUtil.checkEOF(input);
success = true;
} finally {
if (success) {
@@ -327,9 +325,7 @@ final class Lucene40DocValuesReader exte
PagedBytes bytes = new PagedBytes(16);
bytes.copy(input, fixedLength * (long)state.segmentInfo.getDocCount());
final PagedBytes.Reader bytesReader = bytes.freeze(true);
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
- }
+ CodecUtil.checkEOF(input);
success = true;
ramBytesUsed.addAndGet(bytes.ramBytesUsed());
return new BinaryDocValues() {
@@ -367,12 +363,8 @@ final class Lucene40DocValuesReader exte
bytes.copy(data, totalBytes);
final PagedBytes.Reader bytesReader = bytes.freeze(true);
final PackedInts.Reader reader = PackedInts.getReader(index);
- if (data.getFilePointer() != data.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + dataName + "\": read " + data.getFilePointer() + " vs size " + data.length() + " (resource: " + data + ")");
- }
- if (index.getFilePointer() != index.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + indexName + "\": read " + index.getFilePointer() + " vs size " + index.length() + " (resource: " + index + ")");
- }
+ CodecUtil.checkEOF(data);
+ CodecUtil.checkEOF(index);
success = true;
ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
return new BinaryDocValues() {
@@ -414,12 +406,8 @@ final class Lucene40DocValuesReader exte
bytes.copy(data, fixedLength * (long) valueCount);
final PagedBytes.Reader bytesReader = bytes.freeze(true);
final PackedInts.Reader reader = PackedInts.getReader(index);
- if (data.getFilePointer() != data.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + dataName + "\": read " + data.getFilePointer() + " vs size " + data.length() + " (resource: " + data + ")");
- }
- if (index.getFilePointer() != index.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + indexName + "\": read " + index.getFilePointer() + " vs size " + index.length() + " (resource: " + index + ")");
- }
+ CodecUtil.checkEOF(data);
+ CodecUtil.checkEOF(index);
ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
success = true;
return new BinaryDocValues() {
@@ -459,12 +447,8 @@ final class Lucene40DocValuesReader exte
bytes.copy(data, totalBytes);
final PagedBytes.Reader bytesReader = bytes.freeze(true);
final PackedInts.Reader reader = PackedInts.getReader(index);
- if (data.getFilePointer() != data.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + dataName + "\": read " + data.getFilePointer() + " vs size " + data.length() + " (resource: " + data + ")");
- }
- if (index.getFilePointer() != index.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + indexName + "\": read " + index.getFilePointer() + " vs size " + index.length() + " (resource: " + index + ")");
- }
+ CodecUtil.checkEOF(data);
+ CodecUtil.checkEOF(index);
ramBytesUsed.addAndGet(bytes.ramBytesUsed() + reader.ramBytesUsed());
success = true;
return new BinaryDocValues() {
@@ -515,12 +499,8 @@ final class Lucene40DocValuesReader exte
default:
throw new AssertionError();
}
- if (data.getFilePointer() != data.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + dataName + "\": read " + data.getFilePointer() + " vs size " + data.length() + " (resource: " + data + ")");
- }
- if (index.getFilePointer() != index.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + indexName + "\": read " + index.getFilePointer() + " vs size " + index.length() + " (resource: " + index + ")");
- }
+ CodecUtil.checkEOF(data);
+ CodecUtil.checkEOF(index);
success = true;
} finally {
if (success) {
@@ -654,4 +634,8 @@ final class Lucene40DocValuesReader exte
public long ramBytesUsed() {
return ramBytesUsed.get();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ }
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java Fri Apr 4 10:27:05 2014
@@ -107,9 +107,7 @@ class Lucene40FieldInfosReader extends F
omitNorms, storePayloads, indexOptions, oldValuesType.mapping, oldNormsType.mapping, Collections.unmodifiableMap(attributes));
}
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
- }
+ CodecUtil.checkEOF(input);
FieldInfos fieldInfos = new FieldInfos(infos);
success = true;
return fieldInfos;
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java Fri Apr 4 10:27:05 2014
@@ -1168,4 +1168,7 @@ public class Lucene40PostingsReader exte
return 0;
}
+ @Override
+ public void checkIntegrity() throws IOException {}
+
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java Fri Apr 4 10:27:05 2014
@@ -64,9 +64,7 @@ public class Lucene40SegmentInfoReader e
input.readStringStringMap(); // read deprecated attributes
final Set<String> files = input.readStringSet();
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
- }
+ CodecUtil.checkEOF(input);
final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
si.setFiles(files);
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java Fri Apr 4 10:27:05 2014
@@ -34,6 +34,7 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.util.IOUtils;
import java.io.Closeable;
+import java.nio.charset.StandardCharsets;
import static org.apache.lucene.codecs.lucene40.Lucene40StoredFieldsWriter.*;
@@ -193,7 +194,7 @@ public final class Lucene40StoredFieldsR
if ((bits & FIELD_IS_BINARY) != 0) {
visitor.binaryField(info, bytes);
} else {
- visitor.stringField(info, new String(bytes, 0, bytes.length, IOUtils.CHARSET_UTF_8));
+ visitor.stringField(info, new String(bytes, 0, bytes.length, StandardCharsets.UTF_8));
}
}
}
@@ -249,4 +250,7 @@ public final class Lucene40StoredFieldsR
public long ramBytesUsed() {
return 0;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {}
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java Fri Apr 4 10:27:05 2014
@@ -760,5 +760,8 @@ public class Lucene40TermVectorsReader e
public long ramBytesUsed() {
return 0;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {}
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java Fri Apr 4 10:27:05 2014
@@ -132,6 +132,7 @@ import org.apache.lucene.util.packed.Pac
* <li>Header, --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>PackedBlockSize, SingletonDocID --> {@link DataOutput#writeVInt VInt}</li>
* <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --> {@link DataOutput#writeVLong VLong}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>Notes:</p>
* <ul>
@@ -190,7 +191,7 @@ import org.apache.lucene.util.packed.Pac
* each packed or VInt block, when the length of document list is larger than packed block size.</p>
*
* <ul>
- * <li>docFile(.doc) --> Header, <TermFreqs, SkipData?><sup>TermCount</sup></li>
+ * <li>docFile(.doc) --> Header, <TermFreqs, SkipData?><sup>TermCount</sup>, Footer</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>TermFreqs --> <PackedBlock> <sup>PackedDocBlockNum</sup>,
* VIntBlock? </li>
@@ -206,6 +207,7 @@ import org.apache.lucene.util.packed.Pac
* -->
* {@link DataOutput#writeVInt VInt}</li>
* <li>SkipChildLevelPointer --> {@link DataOutput#writeVLong VLong}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>Notes:</p>
* <ul>
@@ -273,7 +275,7 @@ import org.apache.lucene.util.packed.Pac
* <p>The .pos file contains the lists of positions that each term occurs at within documents. It also
* sometimes stores part of payloads and offsets for speedup.</p>
* <ul>
- * <li>PosFile(.pos) --> Header, <TermPositions> <sup>TermCount</sup></li>
+ * <li>PosFile(.pos) --> Header, <TermPositions> <sup>TermCount</sup>, Footer</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>TermPositions --> <PackedPosDeltaBlock> <sup>PackedPosBlockNum</sup>,
* VIntBlock? </li>
@@ -283,6 +285,7 @@ import org.apache.lucene.util.packed.Pac
* <li>PositionDelta, OffsetDelta, OffsetLength -->
* {@link DataOutput#writeVInt VInt}</li>
* <li>PayloadData --> {@link DataOutput#writeByte byte}<sup>PayLength</sup></li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>Notes:</p>
* <ul>
@@ -325,13 +328,14 @@ import org.apache.lucene.util.packed.Pac
* <p>The .pay file will store payloads and offsets associated with certain term-document positions.
* Some payloads and offsets will be separated out into .pos file, for performance reasons.</p>
* <ul>
- * <li>PayFile(.pay): --> Header, <TermPayloads, TermOffsets?> <sup>TermCount</sup></li>
+ * <li>PayFile(.pay): --> Header, <TermPayloads, TermOffsets?> <sup>TermCount</sup>, Footer</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>TermPayloads --> <PackedPayLengthBlock, SumPayLength, PayData> <sup>PackedPayBlockNum</sup>
* <li>TermOffsets --> <PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock> <sup>PackedPayBlockNum</sup>
* <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock --> {@link PackedInts PackedInts}</li>
* <li>SumPayLength --> {@link DataOutput#writeVInt VInt}</li>
* <li>PayData --> {@link DataOutput#writeByte byte}<sup>SumPayLength</sup></li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>Notes:</p>
* <ul>
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java Fri Apr 4 10:27:05 2014
@@ -35,7 +35,6 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -1547,4 +1546,18 @@ public final class Lucene41PostingsReade
return 0;
}
+ @Override
+ public void checkIntegrity() throws IOException {
+ if (version >= Lucene41PostingsWriter.VERSION_CHECKSUM) {
+ if (docIn != null) {
+ CodecUtil.checksumEntireFile(docIn);
+ }
+ if (posIn != null) {
+ CodecUtil.checksumEntireFile(posIn);
+ }
+ if (payIn != null) {
+ CodecUtil.checksumEntireFile(payIn);
+ }
+ }
+ }
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java Fri Apr 4 10:27:05 2014
@@ -64,11 +64,12 @@ public final class Lucene41PostingsWrite
// Increment version to change it
final static int VERSION_START = 0;
final static int VERSION_META_ARRAY = 1;
- final static int VERSION_CURRENT = VERSION_META_ARRAY;
+ final static int VERSION_CHECKSUM = 2;
+ final static int VERSION_CURRENT = VERSION_CHECKSUM;
- final IndexOutput docOut;
- final IndexOutput posOut;
- final IndexOutput payOut;
+ IndexOutput docOut;
+ IndexOutput posOut;
+ IndexOutput payOut;
final static IntBlockTermState emptyState = new IntBlockTermState();
IntBlockTermState lastState;
@@ -113,7 +114,7 @@ public final class Lucene41PostingsWrite
super();
docOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION),
- state.context);
+ state.context);
IndexOutput posOut = null;
IndexOutput payOut = null;
boolean success = false;
@@ -123,7 +124,7 @@ public final class Lucene41PostingsWrite
if (state.fieldInfos.hasProx()) {
posDeltaBuffer = new int[MAX_DATA_SIZE];
posOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene41PostingsFormat.POS_EXTENSION),
- state.context);
+ state.context);
CodecUtil.writeHeader(posOut, POS_CODEC, VERSION_CURRENT);
if (state.fieldInfos.hasPayloads()) {
@@ -144,7 +145,7 @@ public final class Lucene41PostingsWrite
if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
payOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION),
- state.context);
+ state.context);
CodecUtil.writeHeader(payOut, PAY_CODEC, VERSION_CURRENT);
}
} else {
@@ -569,6 +570,26 @@ public final class Lucene41PostingsWrite
@Override
public void close() throws IOException {
- IOUtils.close(docOut, posOut, payOut);
+ // TODO: add a finish() at least to PushBase? DV too...?
+ boolean success = false;
+ try {
+ if (docOut != null) {
+ CodecUtil.writeFooter(docOut);
+ }
+ if (posOut != null) {
+ CodecUtil.writeFooter(posOut);
+ }
+ if (payOut != null) {
+ CodecUtil.writeFooter(payOut);
+ }
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(docOut, posOut, payOut);
+ } else {
+ IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
+ }
+ docOut = posOut = payOut = null;
+ }
}
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java Fri Apr 4 10:27:05 2014
@@ -68,7 +68,7 @@ import org.apache.lucene.util.packed.Blo
* <p>The DocValues metadata or .dvm file.</p>
* <p>For DocValues field, this stores metadata, such as the offset into the
* DocValues data (.dvd)</p>
- * <p>DocValues metadata (.dvm) --> Header,<FieldNumber,EntryType,Entry><sup>NumFields</sup></p>
+ * <p>DocValues metadata (.dvm) --> Header,<FieldNumber,EntryType,Entry><sup>NumFields</sup>,Footer</p>
* <ul>
* <li>Entry --> NumericEntry | BinaryEntry | SortedEntry</li>
* <li>NumericEntry --> DataOffset,CompressionType,PackedVersion</li>
@@ -78,6 +78,7 @@ import org.apache.lucene.util.packed.Blo
* <li>DataOffset,DataLength --> {@link DataOutput#writeLong Int64}</li>
* <li>EntryType,CompressionType --> {@link DataOutput#writeByte Byte}</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>Sorted fields have two entries: a SortedEntry with the FST metadata,
* and an ordinary NumericEntry for the document-to-ord metadata.</p>
@@ -105,7 +106,7 @@ import org.apache.lucene.util.packed.Blo
* <li><a name="dvd" id="dvd"></a>
* <p>The DocValues data or .dvd file.</p>
* <p>For DocValues field, this stores the actual per-document data (the heavy-lifting)</p>
- * <p>DocValues data (.dvd) --> Header,<NumericData | BinaryData | SortedData><sup>NumFields</sup></p>
+ * <p>DocValues data (.dvd) --> Header,<NumericData | BinaryData | SortedData><sup>NumFields</sup>,Footer</p>
* <ul>
* <li>NumericData --> DeltaCompressedNumerics | TableCompressedNumerics | UncompressedNumerics | GCDCompressedNumerics</li>
* <li>BinaryData --> {@link DataOutput#writeByte Byte}<sup>DataLength</sup>,Addresses</li>
@@ -114,6 +115,7 @@ import org.apache.lucene.util.packed.Blo
* <li>TableCompressedNumerics --> TableSize,{@link DataOutput#writeLong Int64}<sup>TableSize</sup>,{@link PackedInts PackedInts}</li>
* <li>UncompressedNumerics --> {@link DataOutput#writeByte Byte}<sup>maxdoc</sup></li>
* <li>Addresses --> {@link MonotonicBlockPackedWriter MonotonicBlockPackedInts(blockSize=4096)}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>SortedSet entries store the list of ordinals in their BinaryData as a
* sequences of increasing {@link DataOutput#writeVLong vLong}s, delta-encoded.</p>
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java Fri Apr 4 10:27:05 2014
@@ -37,6 +37,7 @@ import org.apache.lucene.index.SortedDoc
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -64,6 +65,7 @@ class Lucene42DocValuesProducer extends
private final Map<Integer,BinaryEntry> binaries;
private final Map<Integer,FSTEntry> fsts;
private final IndexInput data;
+ private final int version;
// ram instances we have already loaded
private final Map<Integer,NumericDocValues> numericInstances =
@@ -89,16 +91,16 @@ class Lucene42DocValuesProducer extends
static final int VERSION_START = 0;
static final int VERSION_GCD_COMPRESSION = 1;
- static final int VERSION_CURRENT = VERSION_GCD_COMPRESSION;
+ static final int VERSION_CHECKSUM = 2;
+ static final int VERSION_CURRENT = VERSION_CHECKSUM;
Lucene42DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
maxDoc = state.segmentInfo.getDocCount();
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
// read in the entries from the metadata file.
- IndexInput in = state.directory.openInput(metaName, state.context);
+ ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
boolean success = false;
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
- final int version;
try {
version = CodecUtil.checkHeader(in, metaCodec,
VERSION_START,
@@ -108,8 +110,10 @@ class Lucene42DocValuesProducer extends
fsts = new HashMap<>();
readFields(in, state.fieldInfos);
- if (in.getFilePointer() != in.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + metaName + "\": read " + in.getFilePointer() + " vs size " + in.length() + " (resource: " + in + ")");
+ if (version >= VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(in);
+ } else {
+ CodecUtil.checkEOF(in);
}
success = true;
@@ -199,6 +203,13 @@ class Lucene42DocValuesProducer extends
return ramBytesUsed.get();
}
+ @Override
+ public void checkIntegrity() throws IOException {
+ if (version >= VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(data);
+ }
+ }
+
private NumericDocValues loadNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.number);
data.seek(entry.offset);
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java Fri Apr 4 10:27:05 2014
@@ -92,9 +92,7 @@ final class Lucene42FieldInfosReader ext
omitNorms, storePayloads, indexOptions, docValuesType, normsType, Collections.unmodifiableMap(attributes));
}
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
- }
+ CodecUtil.checkEOF(input);
FieldInfos fieldInfos = new FieldInfos(infos);
success = true;
return fieldInfos;
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsConsumer.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsConsumer.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsConsumer.java Fri Apr 4 10:27:05 2014
@@ -34,14 +34,12 @@ import org.apache.lucene.util.packed.Blo
import org.apache.lucene.util.packed.PackedInts.FormatAndBits;
import org.apache.lucene.util.packed.PackedInts;
+import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.VERSION_CURRENT;
+
/**
* Writer for {@link Lucene42NormsFormat}
*/
-class Lucene42NormsConsumer extends DocValuesConsumer {
- static final int VERSION_START = 0;
- static final int VERSION_GCD_COMPRESSION = 1;
- static final int VERSION_CURRENT = VERSION_GCD_COMPRESSION;
-
+class Lucene42NormsConsumer extends DocValuesConsumer {
static final byte NUMBER = 0;
static final int BLOCK_SIZE = 4096;
@@ -51,7 +49,7 @@ class Lucene42NormsConsumer extends DocV
static final byte UNCOMPRESSED = 2;
static final byte GCD_COMPRESSED = 3;
- final IndexOutput data, meta;
+ IndexOutput data, meta;
final int maxDoc;
final float acceptableOverheadRatio;
@@ -181,6 +179,10 @@ class Lucene42NormsConsumer extends DocV
try {
if (meta != null) {
meta.writeVInt(-1); // write EOF marker
+ CodecUtil.writeFooter(meta); // write checksum
+ }
+ if (data != null) {
+ CodecUtil.writeFooter(data); // write checksum
}
success = true;
} finally {
@@ -189,6 +191,7 @@ class Lucene42NormsConsumer extends DocV
} else {
IOUtils.closeWhileHandlingException(data, meta);
}
+ meta = data = null;
}
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42TermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42TermVectorsFormat.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42TermVectorsFormat.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42TermVectorsFormat.java Fri Apr 4 10:27:05 2014
@@ -59,7 +59,7 @@ import org.apache.lucene.util.packed.Pac
* {@link BlockPackedWriter blocks of packed ints} for positions.</p>
* <p>Here is a more detailed description of the field data file format:</p>
* <ul>
- * <li>VectorData (.tvd) --> <Header>, PackedIntsVersion, ChunkSize, <Chunk><sup>ChunkCount</sup></li>
+ * <li>VectorData (.tvd) --> <Header>, PackedIntsVersion, ChunkSize, <Chunk><sup>ChunkCount</sup>, Footer</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>PackedIntsVersion --> {@link PackedInts#VERSION_CURRENT} as a {@link DataOutput#writeVInt VInt}</li>
* <li>ChunkSize is the number of bytes of terms to accumulate before flushing, as a {@link DataOutput#writeVInt VInt}</li>
@@ -107,14 +107,16 @@ import org.apache.lucene.util.packed.Pac
* <li>FieldTermsAndPayLoads --> Terms (Payloads)</li>
* <li>Terms: term bytes</li>
* <li>Payloads: payload bytes (if the field has payloads)</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* </li>
* <li><a name="vector_index" id="vector_index"></a>
* <p>An index file (extension <tt>.tvx</tt>).</p>
* <ul>
- * <li>VectorIndex (.tvx) --> <Header>, <ChunkIndex></li>
+ * <li>VectorIndex (.tvx) --> <Header>, <ChunkIndex>, Footer</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>ChunkIndex: See {@link CompressingStoredFieldsIndexWriter}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* </li>
* </ol>
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java Fri Apr 4 10:27:05 2014
@@ -66,7 +66,7 @@ public class Lucene45DocValuesConsumer e
* of indirection: docId -> ord. */
public static final int SORTED_SET_SINGLE_VALUED_SORTED = 1;
- final IndexOutput data, meta;
+ IndexOutput data, meta;
final int maxDoc;
/** expert: Creates a new writer */
@@ -438,6 +438,10 @@ public class Lucene45DocValuesConsumer e
try {
if (meta != null) {
meta.writeVInt(-1); // write EOF marker
+ CodecUtil.writeFooter(meta); // write checksum
+ }
+ if (data != null) {
+ CodecUtil.writeFooter(data); // write checksum
}
success = true;
} finally {
@@ -446,6 +450,7 @@ public class Lucene45DocValuesConsumer e
} else {
IOUtils.closeWhileHandlingException(data, meta);
}
+ meta = data = null;
}
}
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesFormat.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesFormat.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesFormat.java Fri Apr 4 10:27:05 2014
@@ -89,7 +89,7 @@ import org.apache.lucene.util.packed.Pac
* <p>The DocValues metadata or .dvm file.</p>
* <p>For DocValues field, this stores metadata, such as the offset into the
* DocValues data (.dvd)</p>
- * <p>DocValues metadata (.dvm) --> Header,<Entry><sup>NumFields</sup></p>
+ * <p>DocValues metadata (.dvm) --> Header,<Entry><sup>NumFields</sup>,Footer</p>
* <ul>
* <li>Entry --> NumericEntry | BinaryEntry | SortedEntry | SortedSetEntry</li>
* <li>NumericEntry --> GCDNumericEntry | TableNumericEntry | DeltaNumericEntry</li>
@@ -109,6 +109,7 @@ import org.apache.lucene.util.packed.Pac
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>MinValue,GCD,MissingOffset,AddressOffset,DataOffset --> {@link DataOutput#writeLong Int64}</li>
* <li>TableSize --> {@link DataOutput#writeVInt vInt}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>Sorted fields have two entries: a BinaryEntry with the value metadata,
* and an ordinary NumericEntry for the document-to-ord metadata.</p>
@@ -138,10 +139,13 @@ import org.apache.lucene.util.packed.Pac
* is written for the addresses.
* <p>MissingOffset points to a byte[] containing a bitset of all documents that had a value for the field.
* If its -1, then there are no missing values.
+ * <p>Checksum contains the CRC32 checksum of all bytes in the .dvm file up
+ * until the checksum. This is used to verify integrity of the file on opening the
+ * index.
* <li><a name="dvd" id="dvd"></a>
* <p>The DocValues data or .dvd file.</p>
* <p>For DocValues field, this stores the actual per-document data (the heavy-lifting)</p>
- * <p>DocValues data (.dvd) --> Header,<NumericData | BinaryData | SortedData><sup>NumFields</sup></p>
+ * <p>DocValues data (.dvd) --> Header,<NumericData | BinaryData | SortedData><sup>NumFields</sup>,Footer</p>
* <ul>
* <li>NumericData --> DeltaCompressedNumerics | TableCompressedNumerics | GCDCompressedNumerics</li>
* <li>BinaryData --> {@link DataOutput#writeByte Byte}<sup>DataLength</sup>,Addresses</li>
@@ -150,6 +154,7 @@ import org.apache.lucene.util.packed.Pac
* <li>TableCompressedNumerics --> {@link PackedInts PackedInts}</li>
* <li>GCDCompressedNumerics --> {@link BlockPackedWriter BlockPackedInts(blockSize=16k)}</li>
* <li>Addresses --> {@link MonotonicBlockPackedWriter MonotonicBlockPackedInts(blockSize=16k)}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>SortedSet entries store the list of ordinals in their BinaryData as a
* sequences of increasing {@link DataOutput#writeVLong vLong}s, delta-encoded.</p>
@@ -179,7 +184,8 @@ public final class Lucene45DocValuesForm
static final String META_EXTENSION = "dvm";
static final int VERSION_START = 0;
static final int VERSION_SORTED_SET_SINGLE_VALUE_OPTIMIZED = 1;
- static final int VERSION_CURRENT = VERSION_SORTED_SET_SINGLE_VALUE_OPTIMIZED;
+ static final int VERSION_CHECKSUM = 2;
+ static final int VERSION_CURRENT = VERSION_CHECKSUM;
static final byte NUMERIC = 0;
static final byte BINARY = 1;
static final byte SORTED = 2;
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java Fri Apr 4 10:27:05 2014
@@ -50,6 +50,7 @@ import org.apache.lucene.index.SortedDoc
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -80,7 +81,7 @@ public class Lucene45DocValuesProducer e
protected Lucene45DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
// read in the entries from the metadata file.
- IndexInput in = state.directory.openInput(metaName, state.context);
+ ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
this.maxDoc = state.segmentInfo.getDocCount();
boolean success = false;
try {
@@ -94,8 +95,10 @@ public class Lucene45DocValuesProducer e
sortedSets = new HashMap<>();
readFields(in, state.fieldInfos);
- if (in.getFilePointer() != in.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + metaName + "\": read " + in.getFilePointer() + " vs size " + in.length() + " (resource: " + in + ")");
+ if (version >= Lucene45DocValuesFormat.VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(in);
+ } else {
+ CodecUtil.checkEOF(in);
}
success = true;
@@ -299,6 +302,13 @@ public class Lucene45DocValuesProducer e
return ramBytesUsed.get();
}
+ @Override
+ public void checkIntegrity() throws IOException {
+ if (version >= Lucene45DocValuesFormat.VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(data);
+ }
+ }
+
LongValues getNumeric(NumericEntry entry) throws IOException {
final IndexInput data = this.data.clone();
data.seek(entry.offset);
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java Fri Apr 4 10:27:05 2014
@@ -32,7 +32,7 @@ import org.apache.lucene.store.DataOutpu
* <p>
* <p>Field names are stored in the field info file, with suffix <tt>.fnm</tt>.</p>
* <p>FieldInfos (.fnm) --> Header,FieldsCount, <FieldName,FieldNumber,
- * FieldBits,DocValuesBits,DocValuesGen,Attributes> <sup>FieldsCount</sup></p>
+ * FieldBits,DocValuesBits,DocValuesGen,Attributes> <sup>FieldsCount</sup>,Footer</p>
* <p>Data types:
* <ul>
* <li>Header --> {@link CodecUtil#checkHeader CodecHeader}</li>
@@ -42,6 +42,7 @@ import org.apache.lucene.store.DataOutpu
* <li>FieldNumber --> {@link DataOutput#writeInt VInt}</li>
* <li>Attributes --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>DocValuesGen --> {@link DataOutput#writeLong(long) Int64}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* </p>
* Field Descriptions:
@@ -113,7 +114,8 @@ public final class Lucene46FieldInfosFor
// Codec header
static final String CODEC_NAME = "Lucene46FieldInfos";
static final int FORMAT_START = 0;
- static final int FORMAT_CURRENT = FORMAT_START;
+ static final int FORMAT_CHECKSUM = 1;
+ static final int FORMAT_CURRENT = FORMAT_CHECKSUM;
// Field flags
static final byte IS_INDEXED = 0x1;
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java Fri Apr 4 10:27:05 2014
@@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -49,13 +50,13 @@ final class Lucene46FieldInfosReader ext
@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
- IndexInput input = directory.openInput(fileName, context);
+ ChecksumIndexInput input = directory.openChecksumInput(fileName, context);
boolean success = false;
try {
- CodecUtil.checkHeader(input, Lucene46FieldInfosFormat.CODEC_NAME,
- Lucene46FieldInfosFormat.FORMAT_START,
- Lucene46FieldInfosFormat.FORMAT_CURRENT);
+ int codecVersion = CodecUtil.checkHeader(input, Lucene46FieldInfosFormat.CODEC_NAME,
+ Lucene46FieldInfosFormat.FORMAT_START,
+ Lucene46FieldInfosFormat.FORMAT_CURRENT);
final int size = input.readVInt(); //read in the size
FieldInfo infos[] = new FieldInfo[size];
@@ -91,9 +92,11 @@ final class Lucene46FieldInfosReader ext
omitNorms, storePayloads, indexOptions, docValuesType, normsType, Collections.unmodifiableMap(attributes));
infos[i].setDocValuesGen(dvGen);
}
-
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
+
+ if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM) {
+ CodecUtil.checkFooter(input);
+ } else {
+ CodecUtil.checkEOF(input);
}
FieldInfos fieldInfos = new FieldInfos(infos);
success = true;
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java Fri Apr 4 10:27:05 2014
@@ -26,9 +26,9 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
/**
@@ -81,6 +81,7 @@ final class Lucene46FieldInfosWriter ext
output.writeLong(fi.getDocValuesGen());
output.writeStringStringMap(fi.attributes());
}
+ CodecUtil.writeFooter(output);
success = true;
} finally {
if (success) {
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java Fri Apr 4 10:27:05 2014
@@ -31,7 +31,7 @@ import org.apache.lucene.store.DataOutpu
* <p>
* Files:
* <ul>
- * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files
+ * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Footer
* </ul>
* </p>
* Data types:
@@ -43,6 +43,7 @@ import org.apache.lucene.store.DataOutpu
* <li>Files --> {@link DataOutput#writeStringSet Set<String>}</li>
* <li>Diagnostics --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* </p>
* Field Descriptions:
@@ -53,9 +54,6 @@ import org.apache.lucene.store.DataOutpu
* <li>IsCompoundFile records whether the segment is written as a compound file or
* not. If this is -1, the segment is not a compound file. If it is 1, the segment
* is a compound file.</li>
- * <li>Checksum contains the CRC32 checksum of all bytes in the segments_N file up
- * until the checksum. This is used to verify integrity of the file on opening the
- * index.</li>
* <li>The Diagnostics Map is privately written by {@link IndexWriter}, as a debugging aid,
* for each segment it creates. It includes metadata like the current Lucene
* version, OS, Java version, why the segment was created (merge, flush,
@@ -89,5 +87,6 @@ public class Lucene46SegmentInfoFormat e
public final static String SI_EXTENSION = "si";
static final String CODEC_NAME = "Lucene46SegmentInfo";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_CHECKSUM = 1;
+ static final int VERSION_CURRENT = VERSION_CHECKSUM;
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java Fri Apr 4 10:27:05 2014
@@ -26,9 +26,9 @@ import org.apache.lucene.codecs.SegmentI
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
/**
@@ -46,12 +46,12 @@ public class Lucene46SegmentInfoReader e
@Override
public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
- final IndexInput input = dir.openInput(fileName, context);
+ final ChecksumIndexInput input = dir.openChecksumInput(fileName, context);
boolean success = false;
try {
- CodecUtil.checkHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME,
- Lucene46SegmentInfoFormat.VERSION_START,
- Lucene46SegmentInfoFormat.VERSION_CURRENT);
+ int codecVersion = CodecUtil.checkHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME,
+ Lucene46SegmentInfoFormat.VERSION_START,
+ Lucene46SegmentInfoFormat.VERSION_CURRENT);
final String version = input.readString();
final int docCount = input.readInt();
if (docCount < 0) {
@@ -61,8 +61,10 @@ public class Lucene46SegmentInfoReader e
final Map<String,String> diagnostics = input.readStringStringMap();
final Set<String> files = input.readStringSet();
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
+ if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(input);
+ } else {
+ CodecUtil.checkEOF(input);
}
final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java Fri Apr 4 10:27:05 2014
@@ -59,7 +59,7 @@ public class Lucene46SegmentInfoWriter e
output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
output.writeStringStringMap(si.getDiagnostics());
output.writeStringSet(si.files());
-
+ CodecUtil.writeFooter(output);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html Fri Apr 4 10:27:05 2014
@@ -383,6 +383,9 @@ on multi-valued fields.</li>
<li>In version 4.5, DocValues were extended to explicitly represent missing values.</li>
<li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
allow updating NumericDocValues fields.</li>
+<li>In version 4.8, checksum footers were added to the end of each index file
+for improved data integrity. Specifically, the last 8 bytes of every index file
+contain the zlib-crc32 checksum of the file.</li>
</ul>
<a name="Limitations" id="Limitations"></a>
<h2>Limitations</h2>
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Fri Apr 4 10:27:05 2014
@@ -310,6 +310,13 @@ public abstract class PerFieldDocValuesF
}
return size;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ for (DocValuesProducer format : formats.values()) {
+ format.checkIntegrity();
+ }
+ }
}
@Override
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Fri Apr 4 10:27:05 2014
@@ -246,6 +246,13 @@ public abstract class PerFieldPostingsFo
}
return sizeInBytes;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ for (FieldsProducer producer : formats.values()) {
+ producer.checkIntegrity();
+ }
+ }
}
@Override
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java Fri Apr 4 10:27:05 2014
@@ -238,4 +238,13 @@ public abstract class AtomicReader exten
* synchronization.
*/
public abstract Bits getLiveDocs();
+
+ /**
+ * Checks consistency of this reader.
+ * <p>
+ * Note that this may be costly in terms of I/O, e.g.
+ * may involve computing a checksum value against large data files.
+ * @lucene.internal
+ */
+ public abstract void checkIntegrity() throws IOException;
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Fri Apr 4 10:27:05 2014
@@ -536,6 +536,10 @@ public class CheckIndex {
reader = new SegmentReader(info, IOContext.DEFAULT);
segInfoStat.openReaderPassed = true;
+
+ if (infoStream != null)
+ infoStream.print(" test: check integrity.........");
+ reader.checkIntegrity();
final int numDocs = reader.numDocs();
toLoseDocCount = numDocs;
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Fri Apr 4 10:27:05 2014
@@ -423,4 +423,9 @@ public class FilterAtomicReader extends
return in.getDocsWithField(field);
}
+ @Override
+ public void checkIntegrity() throws IOException {
+ ensureOpen();
+ in.checkIntegrity();
+ }
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Apr 4 10:27:05 2014
@@ -2085,7 +2085,6 @@ public class IndexWriter implements Clos
rollbackInternal();
}
}
- assert assertEventQueueAfterClose();
}
private void rollbackInternal() throws IOException {
@@ -2180,11 +2179,6 @@ public class IndexWriter implements Clos
}
closed = true;
closing = false;
- try {
- processEvents(false, true);
- } finally {
- notifyAll();
- }
}
}
}
@@ -2657,7 +2651,8 @@ public class IndexWriter implements Clos
false, codec, null);
SegmentMerger merger = new SegmentMerger(mergeReaders, info, infoStream, trackingDir,
- MergeState.CheckAbort.NONE, globalFieldNumberMap, context);
+ MergeState.CheckAbort.NONE, globalFieldNumberMap,
+ context, config.getCheckIntegrityAtMerge());
if (!merger.shouldMerge()) {
return;
@@ -4057,7 +4052,8 @@ public class IndexWriter implements Clos
// OneMerge to return a view over the actual segments to merge
final SegmentMerger merger = new SegmentMerger(merge.getMergeReaders(),
merge.info.info, infoStream, dirWrapper,
- checkAbort, globalFieldNumberMap, context);
+ checkAbort, globalFieldNumberMap,
+ context, config.getCheckIntegrityAtMerge());
merge.checkAborted(directory);
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java Fri Apr 4 10:27:05 2014
@@ -110,6 +110,12 @@ public final class IndexWriterConfig ext
* (set to <code>true</code>). For batch indexing with very large
* ram buffers use <code>false</code> */
public final static boolean DEFAULT_USE_COMPOUND_FILE_SYSTEM = true;
+
+ /** Default value for calling {@link AtomicReader#checkIntegrity()} before
+ * merging segments (set to <code>false</code>). You can set this
+ * to <code>true</code> for additional safety. */
+ public final static boolean DEFAULT_CHECK_INTEGRITY_AT_MERGE = false;
+
/**
* Sets the default (for any instance) maximum time to wait for a write lock
* (in milliseconds).
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java Fri Apr 4 10:27:05 2014
@@ -97,6 +97,9 @@ public class LiveIndexWriterConfig {
/** True if segment flushes should use compound file format */
protected volatile boolean useCompoundFile = IndexWriterConfig.DEFAULT_USE_COMPOUND_FILE_SYSTEM;
+
+ /** True if merging should check integrity of segments before merge */
+ protected volatile boolean checkIntegrityAtMerge = IndexWriterConfig.DEFAULT_CHECK_INTEGRITY_AT_MERGE;
// used by IndexWriterConfig
LiveIndexWriterConfig(Analyzer analyzer, Version matchVersion) {
@@ -152,6 +155,7 @@ public class LiveIndexWriterConfig {
flushPolicy = config.getFlushPolicy();
perThreadHardLimitMB = config.getRAMPerThreadHardLimitMB();
useCompoundFile = config.getUseCompoundFile();
+ checkIntegrityAtMerge = config.getCheckIntegrityAtMerge();
}
/** Returns the default analyzer to use for indexing documents. */
@@ -475,6 +479,26 @@ public class LiveIndexWriterConfig {
return useCompoundFile ;
}
+ /**
+ * Sets if {@link IndexWriter} should call {@link AtomicReader#checkIntegrity()}
+ * on existing segments before merging them into a new one.
+ * <p>
+ * Use <code>true</code> to enable this safety check, which can help
+ * reduce the risk of propagating index corruption from older segments
+ * into new ones, at the expense of slower merging.
+ * </p>
+ */
+ public LiveIndexWriterConfig setCheckIntegrityAtMerge(boolean checkIntegrityAtMerge) {
+ this.checkIntegrityAtMerge = checkIntegrityAtMerge;
+ return this;
+ }
+
+ /** Returns true if {@link AtomicReader#checkIntegrity()} is called before
+ * merging segments. */
+ public boolean getCheckIntegrityAtMerge() {
+ return checkIntegrityAtMerge;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -499,6 +523,7 @@ public class LiveIndexWriterConfig {
sb.append("readerPooling=").append(getReaderPooling()).append("\n");
sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n");
sb.append("useCompoundFile=").append(getUseCompoundFile()).append("\n");
+ sb.append("checkIntegrityAtMerge=").append(getCheckIntegrityAtMerge()).append("\n");
return sb.toString();
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java Fri Apr 4 10:27:05 2014
@@ -299,4 +299,12 @@ public class ParallelAtomicReader extend
NumericDocValues values = reader == null ? null : reader.getNormValues(field);
return values;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ ensureOpen();
+ for (AtomicReader reader : completeReaderSet) {
+ reader.checkIntegrity();
+ }
+ }
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java Fri Apr 4 10:27:05 2014
@@ -95,14 +95,4 @@ final class SegmentDocValues {
IOUtils.reThrow(t);
}
}
-
- /** Returns approximate RAM bytes used. */
- synchronized long ramBytesUsed() {
- long ramBytesUsed = 0;
- for (RefCount<DocValuesProducer> dvp : genDVProducers.values()) {
- ramBytesUsed += dvp.get().ramBytesUsed();
- }
- return ramBytesUsed;
- }
-
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Fri Apr 4 10:27:05 2014
@@ -36,11 +36,9 @@ import org.apache.lucene.codecs.CodecUti
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.ChecksumIndexOutput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.IOUtils;
@@ -69,10 +67,10 @@ import org.apache.lucene.util.IOUtils;
* <p>
* Files:
* <ul>
- * <li><tt>segments.gen</tt>: GenHeader, Generation, Generation
+ * <li><tt>segments.gen</tt>: GenHeader, Generation, Generation, Footer
* <li><tt>segments_N</tt>: Header, Version, NameCounter, SegCount,
* <SegName, SegCodec, DelGen, DeletionCount, FieldInfosGen, UpdatesFiles><sup>SegCount</sup>,
- * CommitUserData, Checksum
+ * CommitUserData, Footer
* </ul>
* </p>
* Data types:
@@ -84,6 +82,7 @@ import org.apache.lucene.util.IOUtils;
* <li>SegName, SegCodec --> {@link DataOutput#writeString String}</li>
* <li>CommitUserData --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>UpdatesFiles --> {@link DataOutput#writeStringSet(Set) Set<String>}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* </p>
* Field Descriptions:
@@ -98,9 +97,6 @@ import org.apache.lucene.util.IOUtils;
* there are no deletes. Anything above zero means there are deletes
* stored by {@link LiveDocsFormat}.</li>
* <li>DeletionCount records the number of deleted documents in this segment.</li>
- * <li>Checksum contains the CRC32 checksum of all bytes in the segments_N file up
- * until the checksum. This is used to verify integrity of the file on opening the
- * index.</li>
* <li>SegCodec is the {@link Codec#getName() name} of the Codec that encoded
* this segment.</li>
* <li>CommitUserData stores an optional user-supplied opaque
@@ -122,10 +118,17 @@ public final class SegmentInfos implemen
/** The file format version for the segments_N codec header, since 4.6+. */
public static final int VERSION_46 = 1;
+
+ /** The file format version for the segments_N codec header, since 4.8+ */
+ public static final int VERSION_48 = 2;
- /** Used for the segments.gen file only!
- * Whenever you add a new format, make it 1 smaller (negative version logic)! */
- public static final int FORMAT_SEGMENTS_GEN_CURRENT = -2;
+ // Used for the segments.gen file only!
+ // Whenever you add a new format, make it 1 smaller (negative version logic)!
+ private static final int FORMAT_SEGMENTS_GEN_47 = -2;
+ private static final int FORMAT_SEGMENTS_GEN_CHECKSUM = -3;
+ private static final int FORMAT_SEGMENTS_GEN_START = FORMAT_SEGMENTS_GEN_47;
+ /** Current format of segments.gen */
+ public static final int FORMAT_SEGMENTS_GEN_CURRENT = FORMAT_SEGMENTS_GEN_CHECKSUM;
/** Used to name new segments. */
public int counter;
@@ -266,6 +269,7 @@ public final class SegmentInfos implemen
genOutput.writeInt(FORMAT_SEGMENTS_GEN_CURRENT);
genOutput.writeLong(generation);
genOutput.writeLong(generation);
+ CodecUtil.writeFooter(genOutput);
} finally {
genOutput.close();
dir.sync(Collections.singleton(IndexFileNames.SEGMENTS_GEN));
@@ -317,7 +321,7 @@ public final class SegmentInfos implemen
lastGeneration = generation;
- ChecksumIndexInput input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ));
+ ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ);
try {
// NOTE: as long as we want to throw indexformattooold (vs corruptindexexception), we need
// to read the magic ourselves.
@@ -326,7 +330,7 @@ public final class SegmentInfos implemen
throw new IndexFormatTooOldException(input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
}
// 4.0+
- int format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_46);
+ int format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_48);
version = input.readLong();
counter = input.readInt();
int numSegments = input.readInt();
@@ -366,10 +370,15 @@ public final class SegmentInfos implemen
}
userData = input.readStringStringMap();
- final long checksumNow = input.getChecksum();
- final long checksumThen = input.readLong();
- if (checksumNow != checksumThen) {
- throw new CorruptIndexException("checksum mismatch in segments file (resource: " + input + ")");
+ if (format >= VERSION_48) {
+ CodecUtil.checkFooter(input);
+ } else {
+ final long checksumNow = input.getChecksum();
+ final long checksumThen = input.readLong();
+ if (checksumNow != checksumThen) {
+ throw new CorruptIndexException("checksum mismatch in segments file (resource: " + input + ")");
+ }
+ CodecUtil.checkEOF(input);
}
success = true;
@@ -402,7 +411,7 @@ public final class SegmentInfos implemen
// Only non-null after prepareCommit has been called and
// before finishCommit is called
- ChecksumIndexOutput pendingSegnOutput;
+ IndexOutput pendingSegnOutput;
private void write(Directory directory) throws IOException {
@@ -415,12 +424,12 @@ public final class SegmentInfos implemen
generation++;
}
- ChecksumIndexOutput segnOutput = null;
+ IndexOutput segnOutput = null;
boolean success = false;
try {
- segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentFileName, IOContext.DEFAULT));
- CodecUtil.writeHeader(segnOutput, "segments", VERSION_46);
+ segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
+ CodecUtil.writeHeader(segnOutput, "segments", VERSION_48);
segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
segnOutput.writeInt(size()); // write infos
@@ -641,9 +650,9 @@ public final class SegmentInfos implemen
// a stale cache (NFS) we have a better chance of
// getting the right generation.
long genB = -1;
- IndexInput genInput = null;
+ ChecksumIndexInput genInput = null;
try {
- genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
+ genInput = directory.openChecksumInput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
} catch (IOException e) {
if (infoStream != null) {
message("segments.gen open: IOException " + e);
@@ -653,18 +662,23 @@ public final class SegmentInfos implemen
if (genInput != null) {
try {
int version = genInput.readInt();
- if (version == FORMAT_SEGMENTS_GEN_CURRENT) {
+ if (version == FORMAT_SEGMENTS_GEN_47 || version == FORMAT_SEGMENTS_GEN_CHECKSUM) {
long gen0 = genInput.readLong();
long gen1 = genInput.readLong();
if (infoStream != null) {
message("fallback check: " + gen0 + "; " + gen1);
}
+ if (version == FORMAT_SEGMENTS_GEN_CHECKSUM) {
+ CodecUtil.checkFooter(genInput);
+ } else {
+ CodecUtil.checkEOF(genInput);
+ }
if (gen0 == gen1) {
// The file is consistent.
genB = gen0;
}
} else {
- throw new IndexFormatTooNewException(genInput, version, FORMAT_SEGMENTS_GEN_CURRENT, FORMAT_SEGMENTS_GEN_CURRENT);
+ throw new IndexFormatTooNewException(genInput, version, FORMAT_SEGMENTS_GEN_START, FORMAT_SEGMENTS_GEN_CURRENT);
}
} catch (IOException err2) {
// rethrow any format exception
@@ -863,7 +877,7 @@ public final class SegmentInfos implemen
}
boolean success = false;
try {
- pendingSegnOutput.finishCommit();
+ CodecUtil.writeFooter(pendingSegnOutput);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Fri Apr 4 10:27:05 2014
@@ -52,7 +52,13 @@ final class SegmentMerger {
// note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!!
SegmentMerger(List<AtomicReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir,
- MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context) throws IOException {
+ MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context, boolean validate) throws IOException {
+ // validate incoming readers
+ if (validate) {
+ for (AtomicReader reader : readers) {
+ reader.checkIntegrity();
+ }
+ }
mergeState = new MergeState(readers, segmentInfo, infoStream, checkAbort);
directory = dir;
this.codec = segmentInfo.getCodec();
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Fri Apr 4 10:27:05 2014
@@ -33,10 +33,13 @@ import org.apache.lucene.util.IOUtils;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
/**
* IndexReader implementation over a single segment.
@@ -72,7 +75,8 @@ public final class SegmentReader extends
}
};
- final Map<String,DocValuesProducer> dvProducers = new HashMap<>();
+ final Map<String,DocValuesProducer> dvProducersByField = new HashMap<>();
+ final Set<DocValuesProducer> dvProducers = Collections.newSetFromMap(new IdentityHashMap<DocValuesProducer,Boolean>());
final FieldInfos fieldInfos;
@@ -177,12 +181,15 @@ public final class SegmentReader extends
// System.out.println("[" + Thread.currentThread().getName() + "] SR.initDocValuesProducers: segInfo=" + si + "; gens=" + genInfos.keySet());
+ // TODO: can we avoid iterating over fieldinfos several times and creating maps of all this stuff if dv updates do not exist?
+
for (Entry<Long,List<FieldInfo>> e : genInfos.entrySet()) {
Long gen = e.getKey();
List<FieldInfo> infos = e.getValue();
DocValuesProducer dvp = segDocValues.getDocValuesProducer(gen, si, IOContext.READ, dir, dvFormat, infos);
for (FieldInfo fi : infos) {
- dvProducers.put(fi.name, dvp);
+ dvProducersByField.put(fi.name, dvp);
+ dvProducers.add(dvp);
}
}
@@ -250,7 +257,7 @@ public final class SegmentReader extends
try {
core.decRef();
} finally {
- dvProducers.clear();
+ dvProducersByField.clear();
try {
IOUtils.close(docValuesLocal, docsWithFieldLocal);
} finally {
@@ -395,13 +402,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Object> dvFields = docValuesLocal.get();
NumericDocValues dvs = (NumericDocValues) dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getNumeric(fi);
dvFields.put(field, dvs);
}
@@ -422,13 +428,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Bits> dvFields = docsWithFieldLocal.get();
Bits dvs = dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getDocsWithField(fi);
dvFields.put(field, dvs);
}
@@ -444,13 +449,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Object> dvFields = docValuesLocal.get();
BinaryDocValues dvs = (BinaryDocValues) dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getBinary(fi);
dvFields.put(field, dvs);
}
@@ -466,13 +470,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Object> dvFields = docValuesLocal.get();
SortedDocValues dvs = (SortedDocValues) dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getSorted(fi);
dvFields.put(field, dvs);
}
@@ -488,13 +491,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Object> dvFields = docValuesLocal.get();
SortedSetDocValues dvs = (SortedSetDocValues) dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getSortedSet(fi);
dvFields.put(field, dvs);
}
@@ -548,12 +550,45 @@ public final class SegmentReader extends
public long ramBytesUsed() {
ensureOpen();
long ramBytesUsed = 0;
- if (segDocValues != null) {
- ramBytesUsed += segDocValues.ramBytesUsed();
+ if (dvProducers != null) {
+ for (DocValuesProducer producer : dvProducers) {
+ ramBytesUsed += producer.ramBytesUsed();
+ }
}
if (core != null) {
ramBytesUsed += core.ramBytesUsed();
}
return ramBytesUsed;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ ensureOpen();
+
+ // stored fields
+ getFieldsReader().checkIntegrity();
+
+ // term vectors
+ TermVectorsReader termVectorsReader = getTermVectorsReader();
+ if (termVectorsReader != null) {
+ termVectorsReader.checkIntegrity();
+ }
+
+ // terms/postings
+ if (core.fields != null) {
+ core.fields.checkIntegrity();
+ }
+
+ // norms
+ if (core.normsProducer != null) {
+ core.normsProducer.checkIntegrity();
+ }
+
+ // docvalues
+ if (dvProducers != null) {
+ for (DocValuesProducer producer : dvProducers) {
+ producer.checkIntegrity();
+ }
+ }
+ }
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Fri Apr 4 10:27:05 2014
@@ -239,4 +239,12 @@ public final class SlowCompositeReaderWr
// TODO: as this is a wrapper, should we really close the delegate?
in.close();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ ensureOpen();
+ for (AtomicReaderContext ctx : in.leaves()) {
+ ctx.reader().checkIntegrity();
+ }
+ }
}
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/Term.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/Term.java?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/Term.java (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/index/Term.java Fri Apr 4 10:27:05 2014
@@ -21,9 +21,9 @@ import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
/**
A Term represents a word from text. This is the unit of search. It is
@@ -83,7 +83,7 @@ public final class Term implements Compa
* the raw bytes will be printed instead. */
public static final String toString(BytesRef termText) {
// the term might not be text, but usually is. so we make a best effort
- CharsetDecoder decoder = IOUtils.CHARSET_UTF_8.newDecoder()
+ CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT);
try {
Modified: lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/search/similarities/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/search/similarities/package.html?rev=1584603&r1=1584602&r2=1584603&view=diff
==============================================================================
--- lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/search/similarities/package.html (original)
+++ lucene/dev/branches/solr5914/lucene/core/src/java/org/apache/lucene/search/similarities/package.html Fri Apr 4 10:27:05 2014
@@ -120,7 +120,7 @@ subclassing the Similarity, one can simp
<h3>Changing {@linkplain org.apache.lucene.search.similarities.DefaultSimilarity}</h3>
<p>
If you are interested in use cases for changing your similarity, see the Lucene users's mailing list at <a
- href="http://www.nabble.com/Overriding-Similarity-tf2128934.html">Overriding Similarity</a>.
+ href="http://www.gossamer-threads.com/lists/lucene/java-user/39125">Overriding Similarity</a>.
In summary, here are a few use cases:
<ol>
<li><p>The <code>SweetSpotSimilarity</code> in