You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/04/01 09:36:48 UTC
svn commit: r1583550 [1/3] - in /lucene/dev/trunk: lucene/
lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/
lucene/codecs/src/java/org/apache/lucene/codecs/memory/
lucene/codecs/src/java...
Author: rmuir
Date: Tue Apr 1 07:36:45 2014
New Revision: 1583550
URL: http://svn.apache.org/r1583550
Log:
LUCENE-2446: add checksums to index files
Added:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java (with props)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksumIndexInput.java (with props)
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java (with props)
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestBufferedChecksum.java (with props)
Removed:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexOutput.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextUtil.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/CodecUtil.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/FieldsProducer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsIndexWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42TermVectorsFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesProducer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/trunk/lucene/misc/build.xml
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestDocSet.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Apr 1 07:36:45 2014
@@ -138,6 +138,13 @@ New Features
* LUCENE-5558: Add TruncateTokenFilter which truncates terms to
the specified length. (Ahmet Arslan via Robert Muir)
+* LUCENE-2446: Added checksums to lucene index files. As of 4.8, the last 8
+ bytes of each file contain a zlib-crc32 checksum. Small metadata files are
+ verified on load. Larger files can be checked on demand via
+ AtomicReader.checkIntegrity. You can configure this to happen automatically
+ before merges by enabling IndexWriterConfig.setCheckIntegrityAtMerge.
+ (Robert Muir)
+
API Changes
* LUCENE-5454: Add RandomAccessOrds, an optional extension of SortedSetDocValues
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java Tue Apr 1 07:36:45 2014
@@ -177,7 +177,10 @@ public class BlockTermsReader extends Fi
}
private void seekDir(IndexInput input, long dirOffset) throws IOException {
- if (version >= BlockTermsWriter.VERSION_APPEND_ONLY) {
+ if (version >= BlockTermsWriter.VERSION_CHECKSUM) {
+ input.seek(input.length() - CodecUtil.footerLength() - 8);
+ dirOffset = input.readLong();
+ } else if (version >= BlockTermsWriter.VERSION_APPEND_ONLY) {
input.seek(input.length() - 8);
dirOffset = input.readLong();
}
@@ -863,4 +866,14 @@ public class BlockTermsReader extends Fi
sizeInBytes += (indexReader!=null) ? indexReader.ramBytesUsed() : 0;
return sizeInBytes;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ // verify terms
+ if (version >= BlockTermsWriter.VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(in);
+ }
+ // verify postings
+ postingsReader.checkIntegrity();
+ }
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java Tue Apr 1 07:36:45 2014
@@ -63,12 +63,13 @@ public class BlockTermsWriter extends Fi
public static final int VERSION_START = 0;
public static final int VERSION_APPEND_ONLY = 1;
public static final int VERSION_META_ARRAY = 2;
- public static final int VERSION_CURRENT = VERSION_META_ARRAY;
+ public static final int VERSION_CHECKSUM = 3;
+ public static final int VERSION_CURRENT = VERSION_CHECKSUM;
/** Extension of terms file */
static final String TERMS_EXTENSION = "tib";
- protected final IndexOutput out;
+ protected IndexOutput out;
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
FieldInfo currentField;
@@ -176,26 +177,30 @@ public class BlockTermsWriter extends Fi
}
public void close() throws IOException {
- try {
- final long dirStart = out.getFilePointer();
-
- out.writeVInt(fields.size());
- for(FieldMetaData field : fields) {
- out.writeVInt(field.fieldInfo.number);
- out.writeVLong(field.numTerms);
- out.writeVLong(field.termsStartPointer);
- if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- out.writeVLong(field.sumTotalTermFreq);
- }
- out.writeVLong(field.sumDocFreq);
- out.writeVInt(field.docCount);
- if (VERSION_CURRENT >= VERSION_META_ARRAY) {
- out.writeVInt(field.longsSize);
+ if (out != null) {
+ try {
+ final long dirStart = out.getFilePointer();
+
+ out.writeVInt(fields.size());
+ for(FieldMetaData field : fields) {
+ out.writeVInt(field.fieldInfo.number);
+ out.writeVLong(field.numTerms);
+ out.writeVLong(field.termsStartPointer);
+ if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
+ out.writeVLong(field.sumTotalTermFreq);
+ }
+ out.writeVLong(field.sumDocFreq);
+ out.writeVInt(field.docCount);
+ if (VERSION_CURRENT >= VERSION_META_ARRAY) {
+ out.writeVInt(field.longsSize);
+ }
}
+ writeTrailer(dirStart);
+ CodecUtil.writeFooter(out);
+ } finally {
+ IOUtils.close(out, postingsWriter, termsIndexWriter);
+ out = null;
}
- writeTrailer(dirStart);
- } finally {
- IOUtils.close(out, postingsWriter, termsIndexWriter);
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java Tue Apr 1 07:36:45 2014
@@ -66,6 +66,8 @@ public class FixedGapTermsIndexReader ex
// start of the field info data
private long dirOffset;
+ private int version;
+
public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, Comparator<BytesRef> termComp, String segmentSuffix, IOContext context)
throws IOException {
@@ -78,6 +80,11 @@ public class FixedGapTermsIndexReader ex
try {
readHeader(in);
+
+ if (version >= FixedGapTermsIndexWriter.VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(in);
+ }
+
indexInterval = in.readVInt();
if (indexInterval < 1) {
throw new CorruptIndexException("invalid indexInterval: " + indexInterval + " (resource=" + in + ")");
@@ -124,7 +131,7 @@ public class FixedGapTermsIndexReader ex
}
private void readHeader(IndexInput input) throws IOException {
- CodecUtil.checkHeader(input, FixedGapTermsIndexWriter.CODEC_NAME,
+ version = CodecUtil.checkHeader(input, FixedGapTermsIndexWriter.CODEC_NAME,
FixedGapTermsIndexWriter.VERSION_CURRENT, FixedGapTermsIndexWriter.VERSION_CURRENT);
}
@@ -273,7 +280,11 @@ public class FixedGapTermsIndexReader ex
public void close() throws IOException {}
private void seekDir(IndexInput input, long dirOffset) throws IOException {
- input.seek(input.length() - 8);
+ if (version >= FixedGapTermsIndexWriter.VERSION_CHECKSUM) {
+ input.seek(input.length() - CodecUtil.footerLength() - 8);
+ } else {
+ input.seek(input.length() - 8);
+ }
dirOffset = input.readLong();
input.seek(dirOffset);
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java Tue Apr 1 07:36:45 2014
@@ -26,7 +26,6 @@ import org.apache.lucene.index.IndexFile
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts;
@@ -43,7 +42,7 @@ import java.io.IOException;
*
* @lucene.experimental */
public class FixedGapTermsIndexWriter extends TermsIndexWriterBase {
- protected final IndexOutput out;
+ protected IndexOutput out;
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tii";
@@ -52,7 +51,8 @@ public class FixedGapTermsIndexWriter ex
final static int VERSION_START = 0;
final static int VERSION_APPEND_ONLY = 1;
final static int VERSION_MONOTONIC_ADDRESSING = 2;
- final static int VERSION_CURRENT = VERSION_MONOTONIC_ADDRESSING;
+ final static int VERSION_CHECKSUM = 3;
+ final static int VERSION_CURRENT = VERSION_CHECKSUM;
final static int BLOCKSIZE = 4096;
final private int termIndexInterval;
@@ -207,38 +207,42 @@ public class FixedGapTermsIndexWriter ex
@Override
public void close() throws IOException {
- boolean success = false;
- try {
- final long dirStart = out.getFilePointer();
- final int fieldCount = fields.size();
-
- int nonNullFieldCount = 0;
- for(int i=0;i<fieldCount;i++) {
- SimpleFieldWriter field = fields.get(i);
- if (field.numIndexTerms > 0) {
- nonNullFieldCount++;
+ if (out != null) {
+ boolean success = false;
+ try {
+ final long dirStart = out.getFilePointer();
+ final int fieldCount = fields.size();
+
+ int nonNullFieldCount = 0;
+ for(int i=0;i<fieldCount;i++) {
+ SimpleFieldWriter field = fields.get(i);
+ if (field.numIndexTerms > 0) {
+ nonNullFieldCount++;
+ }
}
- }
-
- out.writeVInt(nonNullFieldCount);
- for(int i=0;i<fieldCount;i++) {
- SimpleFieldWriter field = fields.get(i);
- if (field.numIndexTerms > 0) {
- out.writeVInt(field.fieldInfo.number);
- out.writeVInt(field.numIndexTerms);
- out.writeVLong(field.termsStart);
- out.writeVLong(field.indexStart);
- out.writeVLong(field.packedIndexStart);
- out.writeVLong(field.packedOffsetsStart);
+
+ out.writeVInt(nonNullFieldCount);
+ for(int i=0;i<fieldCount;i++) {
+ SimpleFieldWriter field = fields.get(i);
+ if (field.numIndexTerms > 0) {
+ out.writeVInt(field.fieldInfo.number);
+ out.writeVInt(field.numIndexTerms);
+ out.writeVLong(field.termsStart);
+ out.writeVLong(field.indexStart);
+ out.writeVLong(field.packedIndexStart);
+ out.writeVLong(field.packedOffsetsStart);
+ }
}
- }
- writeTrailer(dirStart);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(out);
- } else {
- IOUtils.closeWhileHandlingException(out);
+ writeTrailer(dirStart);
+ CodecUtil.writeFooter(out);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(out);
+ } else {
+ IOUtils.closeWhileHandlingException(out);
+ }
+ out = null;
}
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java Tue Apr 1 07:36:45 2014
@@ -62,6 +62,10 @@ public class VariableGapTermsIndexReader
try {
version = readHeader(in);
+
+ if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(in);
+ }
seekDir(in, dirOffset);
@@ -190,7 +194,10 @@ public class VariableGapTermsIndexReader
public void close() throws IOException {}
private void seekDir(IndexInput input, long dirOffset) throws IOException {
- if (version >= VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) {
+ if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) {
+ input.seek(input.length() - CodecUtil.footerLength() - 8);
+ dirOffset = input.readLong();
+ } else if (version >= VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) {
input.seek(input.length() - 8);
dirOffset = input.readLong();
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java Tue Apr 1 07:36:45 2014
@@ -45,7 +45,7 @@ import org.apache.lucene.util.fst.Util;
*
* @lucene.experimental */
public class VariableGapTermsIndexWriter extends TermsIndexWriterBase {
- protected final IndexOutput out;
+ protected IndexOutput out;
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tiv";
@@ -53,7 +53,8 @@ public class VariableGapTermsIndexWriter
final static String CODEC_NAME = "VARIABLE_GAP_TERMS_INDEX";
final static int VERSION_START = 0;
final static int VERSION_APPEND_ONLY = 1;
- final static int VERSION_CURRENT = VERSION_APPEND_ONLY;
+ final static int VERSION_CHECKSUM = 2;
+ final static int VERSION_CURRENT = VERSION_CHECKSUM;
private final List<FSTFieldWriter> fields = new ArrayList<>();
@@ -290,30 +291,34 @@ public class VariableGapTermsIndexWriter
@Override
public void close() throws IOException {
- try {
- final long dirStart = out.getFilePointer();
- final int fieldCount = fields.size();
-
- int nonNullFieldCount = 0;
- for(int i=0;i<fieldCount;i++) {
- FSTFieldWriter field = fields.get(i);
- if (field.fst != null) {
- nonNullFieldCount++;
- }
- }
-
- out.writeVInt(nonNullFieldCount);
- for(int i=0;i<fieldCount;i++) {
- FSTFieldWriter field = fields.get(i);
- if (field.fst != null) {
- out.writeVInt(field.fieldInfo.number);
- out.writeVLong(field.indexStart);
+ if (out != null) {
+ try {
+ final long dirStart = out.getFilePointer();
+ final int fieldCount = fields.size();
+
+ int nonNullFieldCount = 0;
+ for(int i=0;i<fieldCount;i++) {
+ FSTFieldWriter field = fields.get(i);
+ if (field.fst != null) {
+ nonNullFieldCount++;
+ }
+ }
+
+ out.writeVInt(nonNullFieldCount);
+ for(int i=0;i<fieldCount;i++) {
+ FSTFieldWriter field = fields.get(i);
+ if (field.fst != null) {
+ out.writeVInt(field.fieldInfo.number);
+ out.writeVLong(field.indexStart);
+ }
+ }
+ writeTrailer(dirStart);
+ CodecUtil.writeFooter(out);
+ } finally {
+ out.close();
+ out = null;
}
}
- writeTrailer(dirStart);
- } finally {
- out.close();
- }
}
private void writeTrailer(long dirStart) throws IOException {
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Tue Apr 1 07:36:45 2014
@@ -39,8 +39,8 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -66,7 +66,7 @@ import org.apache.lucene.util.automaton.
* </p>
* <ul>
* <li>BloomFilter (.blm) --> Header, DelegatePostingsFormatName,
- * NumFilteredFields, Filter<sup>NumFilteredFields</sup></li>
+ * NumFilteredFields, Filter<sup>NumFilteredFields</sup>, Footer</li>
* <li>Filter --> FieldNumber, FuzzySet</li>
* <li>FuzzySet -->See {@link FuzzySet#serialize(DataOutput)}</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
@@ -75,13 +75,16 @@ import org.apache.lucene.util.automaton.
* <li>NumFilteredFields --> {@link DataOutput#writeInt Uint32}</li>
* <li>FieldNumber --> {@link DataOutput#writeInt Uint32} The number of the
* field in this segment</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* @lucene.experimental
*/
public final class BloomFilteringPostingsFormat extends PostingsFormat {
public static final String BLOOM_CODEC_NAME = "BloomFilter";
- public static final int BLOOM_CODEC_VERSION = 1;
+ public static final int VERSION_START = 1;
+ public static final int VERSION_CHECKSUM = 2;
+ public static final int VERSION_CURRENT = VERSION_CHECKSUM;
/** Extension of Bloom Filters file */
static final String BLOOM_EXTENSION = "blm";
@@ -157,12 +160,11 @@ public final class BloomFilteringPosting
String bloomFileName = IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION);
- IndexInput bloomIn = null;
+ ChecksumIndexInput bloomIn = null;
boolean success = false;
try {
- bloomIn = state.directory.openInput(bloomFileName, state.context);
- CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, BLOOM_CODEC_VERSION,
- BLOOM_CODEC_VERSION);
+ bloomIn = state.directory.openChecksumInput(bloomFileName, state.context);
+ int version = CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT);
// // Load the hash function used in the BloomFilter
// hashFunction = HashFunction.forName(bloomIn.readString());
// Load the delegate postings format
@@ -178,6 +180,11 @@ public final class BloomFilteringPosting
FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNum);
bloomsByFieldName.put(fieldInfo.name, bloom);
}
+ if (version >= VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(bloomIn);
+ } else {
+ CodecUtil.checkEOF(bloomIn);
+ }
IOUtils.close(bloomIn);
success = true;
} finally {
@@ -390,6 +397,11 @@ public final class BloomFilteringPosting
}
return sizeInBytes;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ delegateFieldsProducer.checkIntegrity();
+ }
}
class BloomFilteredFieldsConsumer extends FieldsConsumer {
@@ -466,10 +478,8 @@ public final class BloomFilteringPosting
state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION);
IndexOutput bloomOutput = null;
try {
- bloomOutput = state.directory
- .createOutput(bloomFileName, state.context);
- CodecUtil.writeHeader(bloomOutput, BLOOM_CODEC_NAME,
- BLOOM_CODEC_VERSION);
+ bloomOutput = state.directory.createOutput(bloomFileName, state.context);
+ CodecUtil.writeHeader(bloomOutput, BLOOM_CODEC_NAME, VERSION_CURRENT);
// remember the name of the postings format we will delegate to
bloomOutput.writeString(delegatePostingsFormat.getName());
@@ -481,6 +491,7 @@ public final class BloomFilteringPosting
bloomOutput.writeInt(fieldInfo.number);
saveAppropriatelySizedBloomFilter(bloomOutput, bloomFilter, fieldInfo);
}
+ CodecUtil.writeFooter(bloomOutput);
} finally {
IOUtils.close(bloomOutput);
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java Tue Apr 1 07:36:45 2014
@@ -40,7 +40,7 @@ import static org.apache.lucene.codecs.m
*/
class DirectDocValuesConsumer extends DocValuesConsumer {
- final IndexOutput data, meta;
+ IndexOutput data, meta;
final int maxDoc;
DirectDocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
@@ -142,6 +142,10 @@ class DirectDocValuesConsumer extends Do
try {
if (meta != null) {
meta.writeVInt(-1); // write EOF marker
+ CodecUtil.writeFooter(meta); // write checksum
+ }
+ if (data != null) {
+ CodecUtil.writeFooter(data);
}
success = true;
} finally {
@@ -150,6 +154,7 @@ class DirectDocValuesConsumer extends Do
} else {
IOUtils.closeWhileHandlingException(data, meta);
}
+ data = meta = null;
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java Tue Apr 1 07:36:45 2014
@@ -33,6 +33,7 @@ import org.apache.lucene.index.RandomAcc
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -65,6 +66,7 @@ class DirectDocValuesProducer extends Do
private final int maxDoc;
private final AtomicLong ramBytesUsed;
+ private final int version;
static final byte NUMBER = 0;
static final byte BYTES = 1;
@@ -72,22 +74,27 @@ class DirectDocValuesProducer extends Do
static final byte SORTED_SET = 3;
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_CHECKSUM = 1;
+ static final int VERSION_CURRENT = VERSION_CHECKSUM;
DirectDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
maxDoc = state.segmentInfo.getDocCount();
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
// read in the entries from the metadata file.
- IndexInput in = state.directory.openInput(metaName, state.context);
+ ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
boolean success = false;
- final int version;
try {
version = CodecUtil.checkHeader(in, metaCodec,
VERSION_START,
VERSION_CURRENT);
readFields(in);
+ if (version >= VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(in);
+ } else {
+ CodecUtil.checkEOF(in);
+ }
success = true;
} finally {
if (success) {
@@ -186,6 +193,13 @@ class DirectDocValuesProducer extends Do
}
@Override
+ public void checkIntegrity() throws IOException {
+ if (version >= VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(data);
+ }
+ }
+
+ @Override
public synchronized NumericDocValues getNumeric(FieldInfo field) throws IOException {
NumericDocValues instance = numericInstances.get(field.number);
if (instance == null) {
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Tue Apr 1 07:36:45 2014
@@ -109,6 +109,7 @@ public final class DirectPostingsFormat
if (state.context.context != IOContext.Context.MERGE) {
FieldsProducer loadedPostings;
try {
+ postings.checkIntegrity();
loadedPostings = new DirectFields(state, postings, minSkipCount, lowFreqCutoff);
} finally {
postings.close();
@@ -157,6 +158,12 @@ public final class DirectPostingsFormat
}
return sizeInBytes;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ // if we read entirely into ram, we already validated.
+ // otherwise returned the raw postings reader
+ }
}
private final static class DirectField extends Terms {
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java Tue Apr 1 07:36:45 2014
@@ -38,6 +38,7 @@ import org.apache.lucene.index.TermState
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.automaton.ByteRunAutomaton;
@@ -56,14 +57,13 @@ import org.apache.lucene.codecs.BlockTer
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.memory.FSTTermsReader.TermsReader;
/**
* FST-based terms dictionary reader.
*
* The FST index maps each term and its ord, and during seek
* the ord is used fetch metadata from a single block.
- * The term dictionary is fully memeory resident.
+ * The term dictionary is fully memory resident.
*
* @lucene.experimental
*/
@@ -71,8 +71,7 @@ public class FSTOrdTermsReader extends F
static final int INTERVAL = FSTOrdTermsWriter.SKIP_INTERVAL;
final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader;
- IndexInput indexIn = null;
- IndexInput blockIn = null;
+ int version;
//static final boolean TEST = false;
public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
@@ -80,11 +79,18 @@ public class FSTOrdTermsReader extends F
final String termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTOrdTermsWriter.TERMS_BLOCK_EXTENSION);
this.postingsReader = postingsReader;
+ ChecksumIndexInput indexIn = null;
+ IndexInput blockIn = null;
+ boolean success = false;
try {
- this.indexIn = state.directory.openInput(termsIndexFileName, state.context);
- this.blockIn = state.directory.openInput(termsBlockFileName, state.context);
- readHeader(indexIn);
+ indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context);
+ blockIn = state.directory.openInput(termsBlockFileName, state.context);
+ version = readHeader(indexIn);
readHeader(blockIn);
+ if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(blockIn);
+ }
+
this.postingsReader.init(blockIn);
seekDir(blockIn);
@@ -100,12 +106,22 @@ public class FSTOrdTermsReader extends F
int longsSize = blockIn.readVInt();
FST<Long> index = new FST<>(indexIn, PositiveIntOutputs.getSingleton());
- TermsReader current = new TermsReader(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, index);
+ TermsReader current = new TermsReader(fieldInfo, blockIn, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize, index);
TermsReader previous = fields.put(fieldInfo.name, current);
- checkFieldSummary(state.segmentInfo, current, previous);
+ checkFieldSummary(state.segmentInfo, indexIn, blockIn, current, previous);
+ }
+ if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(indexIn);
+ } else {
+ CodecUtil.checkEOF(indexIn);
}
+ success = true;
} finally {
- IOUtils.closeWhileHandlingException(indexIn, blockIn);
+ if (success) {
+ IOUtils.close(indexIn, blockIn);
+ } else {
+ IOUtils.closeWhileHandlingException(indexIn, blockIn);
+ }
}
}
@@ -115,10 +131,14 @@ public class FSTOrdTermsReader extends F
FSTOrdTermsWriter.TERMS_VERSION_CURRENT);
}
private void seekDir(IndexInput in) throws IOException {
- in.seek(in.length() - 8);
+ if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
+ in.seek(in.length() - CodecUtil.footerLength() - 8);
+ } else {
+ in.seek(in.length() - 8);
+ }
in.seek(in.readLong());
}
- private void checkFieldSummary(SegmentInfo info, TermsReader field, TermsReader previous) throws IOException {
+ private void checkFieldSummary(SegmentInfo info, IndexInput indexIn, IndexInput blockIn, TermsReader field, TermsReader previous) throws IOException {
// #docs with field must be <= #docs
if (field.docCount < 0 || field.docCount > info.getDocCount()) {
throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.getDocCount() + " (resource=" + indexIn + ", " + blockIn + ")");
@@ -176,7 +196,7 @@ public class FSTOrdTermsReader extends F
final byte[] metaLongsBlock;
final byte[] metaBytesBlock;
- TermsReader(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST<Long> index) throws IOException {
+ TermsReader(FieldInfo fieldInfo, IndexInput blockIn, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize, FST<Long> index) throws IOException {
this.fieldInfo = fieldInfo;
this.numTerms = numTerms;
this.sumTotalTermFreq = sumTotalTermFreq;
@@ -819,4 +839,9 @@ public class FSTOrdTermsReader extends F
}
return ramBytesUsed;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ postingsReader.checkIntegrity();
+ }
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java Tue Apr 1 07:36:45 2014
@@ -73,9 +73,10 @@ import org.apache.lucene.util.fst.Util;
* </p>
*
* <ul>
- * <li>TermIndex(.tix) --> Header, TermFST<sup>NumFields</sup></li>
+ * <li>TermIndex(.tix) --> Header, TermFST<sup>NumFields</sup>, Footer</li>
* <li>TermFST --> {@link FST FST<long>}</li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
*
* <p>Notes:</p>
@@ -103,7 +104,7 @@ import org.apache.lucene.util.fst.Util;
* <ul>
* <li>TermBlock(.tbk) --> Header, <i>PostingsHeader</i>, FieldSummary, DirOffset</li>
* <li>FieldSummary --> NumFields, <FieldNumber, NumTerms, SumTotalTermFreq?, SumDocFreq,
- * DocCount, LongsSize, DataBlock > <sup>NumFields</sup></li>
+ * DocCount, LongsSize, DataBlock > <sup>NumFields</sup>, Footer</li>
*
* <li>DataBlock --> StatsBlockLength, MetaLongsBlockLength, MetaBytesBlockLength,
* SkipBlock, StatsBlock, MetaLongsBlock, MetaBytesBlock </li>
@@ -119,6 +120,7 @@ import org.apache.lucene.util.fst.Util;
* <li>NumTerms, SumTotalTermFreq, SumDocFreq, StatsBlockLength, MetaLongsBlockLength, MetaBytesBlockLength,
* StatsFPDelta, MetaLongsSkipFPDelta, MetaBytesSkipFPDelta, MetaLongsSkipStart, TotalTermFreq,
* LongDelta,--> {@link DataOutput#writeVLong VLong}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>Notes: </p>
* <ul>
@@ -148,7 +150,8 @@ public class FSTOrdTermsWriter extends F
static final String TERMS_BLOCK_EXTENSION = "tbk";
static final String TERMS_CODEC_NAME = "FST_ORD_TERMS_DICT";
public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START;
+ public static final int TERMS_VERSION_CHECKSUM = 1;
+ public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
public static final int SKIP_INTERVAL = 8;
final PostingsWriterBase postingsWriter;
@@ -218,36 +221,41 @@ public class FSTOrdTermsWriter extends F
}
public void close() throws IOException {
- IOException ioe = null;
- try {
- final long blockDirStart = blockOut.getFilePointer();
-
- // write field summary
- blockOut.writeVInt(fields.size());
- for (FieldMetaData field : fields) {
- blockOut.writeVInt(field.fieldInfo.number);
- blockOut.writeVLong(field.numTerms);
- if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- blockOut.writeVLong(field.sumTotalTermFreq);
+ if (blockOut != null) {
+ IOException ioe = null;
+ try {
+ final long blockDirStart = blockOut.getFilePointer();
+
+ // write field summary
+ blockOut.writeVInt(fields.size());
+ for (FieldMetaData field : fields) {
+ blockOut.writeVInt(field.fieldInfo.number);
+ blockOut.writeVLong(field.numTerms);
+ if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
+ blockOut.writeVLong(field.sumTotalTermFreq);
+ }
+ blockOut.writeVLong(field.sumDocFreq);
+ blockOut.writeVInt(field.docCount);
+ blockOut.writeVInt(field.longsSize);
+ blockOut.writeVLong(field.statsOut.getFilePointer());
+ blockOut.writeVLong(field.metaLongsOut.getFilePointer());
+ blockOut.writeVLong(field.metaBytesOut.getFilePointer());
+
+ field.skipOut.writeTo(blockOut);
+ field.statsOut.writeTo(blockOut);
+ field.metaLongsOut.writeTo(blockOut);
+ field.metaBytesOut.writeTo(blockOut);
+ field.dict.save(indexOut);
}
- blockOut.writeVLong(field.sumDocFreq);
- blockOut.writeVInt(field.docCount);
- blockOut.writeVInt(field.longsSize);
- blockOut.writeVLong(field.statsOut.getFilePointer());
- blockOut.writeVLong(field.metaLongsOut.getFilePointer());
- blockOut.writeVLong(field.metaBytesOut.getFilePointer());
-
- field.skipOut.writeTo(blockOut);
- field.statsOut.writeTo(blockOut);
- field.metaLongsOut.writeTo(blockOut);
- field.metaBytesOut.writeTo(blockOut);
- field.dict.save(indexOut);
+ writeTrailer(blockOut, blockDirStart);
+ CodecUtil.writeFooter(indexOut);
+ CodecUtil.writeFooter(blockOut);
+ } catch (IOException ioe2) {
+ ioe = ioe2;
+ } finally {
+ IOUtils.closeWhileHandlingException(ioe, blockOut, indexOut, postingsWriter);
+ blockOut = null;
}
- writeTrailer(blockOut, blockDirStart);
- } catch (IOException ioe2) {
- ioe = ioe2;
- } finally {
- IOUtils.closeWhileHandlingException(ioe, blockOut, indexOut, postingsWriter);
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java Tue Apr 1 07:36:45 2014
@@ -59,7 +59,7 @@ import org.apache.lucene.codecs.CodecUti
* FST-based terms dictionary reader.
*
* The FST directly maps each term and its metadata,
- * it is memeory resident.
+ * it is memory resident.
*
* @lucene.experimental
*/
@@ -67,18 +67,21 @@ import org.apache.lucene.codecs.CodecUti
public class FSTTermsReader extends FieldsProducer {
final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader;
- final IndexInput in;
//static boolean TEST = false;
+ final int version;
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
this.postingsReader = postingsReader;
- this.in = state.directory.openInput(termsFileName, state.context);
+ final IndexInput in = state.directory.openInput(termsFileName, state.context);
boolean success = false;
try {
- readHeader(in);
+ version = readHeader(in);
+ if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(in);
+ }
this.postingsReader.init(in);
seekDir(in);
@@ -92,13 +95,15 @@ public class FSTTermsReader extends Fiel
long sumDocFreq = in.readVLong();
int docCount = in.readVInt();
int longsSize = in.readVInt();
- TermsReader current = new TermsReader(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize);
+ TermsReader current = new TermsReader(fieldInfo, in, numTerms, sumTotalTermFreq, sumDocFreq, docCount, longsSize);
TermsReader previous = fields.put(fieldInfo.name, current);
- checkFieldSummary(state.segmentInfo, current, previous);
+ checkFieldSummary(state.segmentInfo, in, current, previous);
}
success = true;
} finally {
- if (!success) {
+ if (success) {
+ IOUtils.close(in);
+ } else {
IOUtils.closeWhileHandlingException(in);
}
}
@@ -110,10 +115,14 @@ public class FSTTermsReader extends Fiel
FSTTermsWriter.TERMS_VERSION_CURRENT);
}
private void seekDir(IndexInput in) throws IOException {
- in.seek(in.length() - 8);
+ if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
+ in.seek(in.length() - CodecUtil.footerLength() - 8);
+ } else {
+ in.seek(in.length() - 8);
+ }
in.seek(in.readLong());
}
- private void checkFieldSummary(SegmentInfo info, TermsReader field, TermsReader previous) throws IOException {
+ private void checkFieldSummary(SegmentInfo info, IndexInput in, TermsReader field, TermsReader previous) throws IOException {
// #docs with field must be <= #docs
if (field.docCount < 0 || field.docCount > info.getDocCount()) {
throw new CorruptIndexException("invalid docCount: " + field.docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")");
@@ -150,7 +159,7 @@ public class FSTTermsReader extends Fiel
@Override
public void close() throws IOException {
try {
- IOUtils.close(in, postingsReader);
+ IOUtils.close(postingsReader);
} finally {
fields.clear();
}
@@ -165,7 +174,7 @@ public class FSTTermsReader extends Fiel
final int longsSize;
final FST<FSTTermOutputs.TermData> dict;
- TermsReader(FieldInfo fieldInfo, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) throws IOException {
+ TermsReader(FieldInfo fieldInfo, IndexInput in, long numTerms, long sumTotalTermFreq, long sumDocFreq, int docCount, int longsSize) throws IOException {
this.fieldInfo = fieldInfo;
this.numTerms = numTerms;
this.sumTotalTermFreq = sumTotalTermFreq;
@@ -729,4 +738,9 @@ public class FSTTermsReader extends Fiel
}
return ramBytesUsed;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ postingsReader.checkIntegrity();
+ }
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java Tue Apr 1 07:36:45 2014
@@ -124,11 +124,12 @@ public class FSTTermsWriter extends Fiel
static final String TERMS_EXTENSION = "tmp";
static final String TERMS_CODEC_NAME = "FST_TERMS_DICT";
public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START;
+ public static final int TERMS_VERSION_CHECKSUM = 1;
+ public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
- final IndexOutput out;
+ IndexOutput out;
final int maxDoc;
final List<FieldMetaData> fields = new ArrayList<>();
@@ -199,28 +200,32 @@ public class FSTTermsWriter extends Fiel
}
public void close() throws IOException {
- IOException ioe = null;
- try {
- // write field summary
- final long dirStart = out.getFilePointer();
-
- out.writeVInt(fields.size());
- for (FieldMetaData field : fields) {
- out.writeVInt(field.fieldInfo.number);
- out.writeVLong(field.numTerms);
- if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- out.writeVLong(field.sumTotalTermFreq);
+ if (out != null) {
+ IOException ioe = null;
+ try {
+ // write field summary
+ final long dirStart = out.getFilePointer();
+
+ out.writeVInt(fields.size());
+ for (FieldMetaData field : fields) {
+ out.writeVInt(field.fieldInfo.number);
+ out.writeVLong(field.numTerms);
+ if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
+ out.writeVLong(field.sumTotalTermFreq);
+ }
+ out.writeVLong(field.sumDocFreq);
+ out.writeVInt(field.docCount);
+ out.writeVInt(field.longsSize);
+ field.dict.save(out);
}
- out.writeVLong(field.sumDocFreq);
- out.writeVInt(field.docCount);
- out.writeVInt(field.longsSize);
- field.dict.save(out);
+ writeTrailer(out, dirStart);
+ CodecUtil.writeFooter(out);
+ } catch (IOException ioe2) {
+ ioe = ioe2;
+ } finally {
+ IOUtils.closeWhileHandlingException(ioe, out, postingsWriter);
+ out = null;
}
- writeTrailer(out, dirStart);
- } catch (IOException ioe2) {
- ioe = ioe2;
- } finally {
- IOUtils.closeWhileHandlingException(ioe, out, postingsWriter);
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java Tue Apr 1 07:36:45 2014
@@ -59,7 +59,7 @@ import static org.apache.lucene.codecs.m
* Writer for {@link MemoryDocValuesFormat}
*/
class MemoryDocValuesConsumer extends DocValuesConsumer {
- final IndexOutput data, meta;
+ IndexOutput data, meta;
final int maxDoc;
final float acceptableOverheadRatio;
@@ -208,6 +208,10 @@ class MemoryDocValuesConsumer extends Do
try {
if (meta != null) {
meta.writeVInt(-1); // write EOF marker
+ CodecUtil.writeFooter(meta); // write checksum
+ }
+ if (data != null) {
+ CodecUtil.writeFooter(data);
}
success = true;
} finally {
@@ -216,6 +220,7 @@ class MemoryDocValuesConsumer extends Do
} else {
IOUtils.closeWhileHandlingException(data, meta);
}
+ data = meta = null;
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java Tue Apr 1 07:36:45 2014
@@ -37,6 +37,7 @@ import org.apache.lucene.index.SortedDoc
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -77,6 +78,7 @@ class MemoryDocValuesProducer extends Do
private final int maxDoc;
private final AtomicLong ramBytesUsed;
+ private final int version;
static final byte NUMBER = 0;
static final byte BYTES = 1;
@@ -91,15 +93,15 @@ class MemoryDocValuesProducer extends Do
static final int VERSION_START = 0;
static final int VERSION_GCD_COMPRESSION = 1;
- static final int VERSION_CURRENT = VERSION_GCD_COMPRESSION;
+ static final int VERSION_CHECKSUM = 2;
+ static final int VERSION_CURRENT = VERSION_CHECKSUM;
MemoryDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
maxDoc = state.segmentInfo.getDocCount();
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
// read in the entries from the metadata file.
- IndexInput in = state.directory.openInput(metaName, state.context);
+ ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
boolean success = false;
- final int version;
try {
version = CodecUtil.checkHeader(in, metaCodec,
VERSION_START,
@@ -108,6 +110,11 @@ class MemoryDocValuesProducer extends Do
binaries = new HashMap<>();
fsts = new HashMap<>();
readFields(in, state.fieldInfos);
+ if (version >= VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(in);
+ } else {
+ CodecUtil.checkEOF(in);
+ }
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
success = true;
} finally {
@@ -208,6 +215,13 @@ class MemoryDocValuesProducer extends Do
return ramBytesUsed.get();
}
+ @Override
+ public void checkIntegrity() throws IOException {
+ if (version >= VERSION_CHECKSUM) {
+ CodecUtil.checksumEntireFile(data);
+ }
+ }
+
private NumericDocValues loadNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.number);
data.seek(entry.offset + entry.missingBytes);
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Tue Apr 1 07:36:45 2014
@@ -25,6 +25,7 @@ import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
+import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
@@ -41,6 +42,7 @@ import org.apache.lucene.index.SegmentWr
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -271,6 +273,9 @@ public final class MemoryPostingsFormat
}
private static String EXTENSION = "ram";
+ private static final String CODEC_NAME = "MemoryPostings";
+ private static final int VERSION_START = 0;
+ private static final int VERSION_CURRENT = VERSION_START;
private class MemoryFieldsConsumer extends FieldsConsumer implements Closeable {
private final SegmentWriteState state;
@@ -279,6 +284,7 @@ public final class MemoryPostingsFormat
private MemoryFieldsConsumer(SegmentWriteState state) throws IOException {
final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
out = state.directory.createOutput(fileName, state.context);
+ CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
this.state = state;
}
@@ -403,6 +409,7 @@ public final class MemoryPostingsFormat
// EOF marker:
try {
out.writeVInt(0);
+ CodecUtil.writeFooter(out);
} finally {
out.close();
}
@@ -951,7 +958,8 @@ public final class MemoryPostingsFormat
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- final IndexInput in = state.directory.openInput(fileName, IOContext.READONCE);
+ final ChecksumIndexInput in = state.directory.openChecksumInput(fileName, IOContext.READONCE);
+ CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
final SortedMap<String,TermsReader> fields = new TreeMap<>();
@@ -965,6 +973,7 @@ public final class MemoryPostingsFormat
// System.out.println("load field=" + termsReader.field.name);
fields.put(termsReader.field.name, termsReader);
}
+ CodecUtil.checkFooter(in);
} finally {
in.close();
}
@@ -1002,6 +1011,9 @@ public final class MemoryPostingsFormat
}
return sizeInBytes;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {}
};
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java Tue Apr 1 07:36:45 2014
@@ -653,4 +653,9 @@ public class PulsingPostingsReader exten
public long ramBytesUsed() {
return ((wrappedPostingsReader!=null) ? wrappedPostingsReader.ramBytesUsed(): 0);
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ wrappedPostingsReader.checkIntegrity();
+ }
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java Tue Apr 1 07:36:45 2014
@@ -706,4 +706,9 @@ public class SepPostingsReader extends P
public long ramBytesUsed() {
return 0;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ // TODO: remove sep layout, its fallen behind on features...
+ }
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesReader.java Tue Apr 1 07:36:45 2014
@@ -17,6 +17,7 @@ package org.apache.lucene.codecs.simplet
* limitations under the License.
*/
+import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.CHECKSUM;
import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.END;
import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.FIELD;
import static org.apache.lucene.codecs.simpletext.SimpleTextDocValuesWriter.LENGTH;
@@ -48,6 +49,8 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.store.BufferedChecksumIndexInput;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -468,4 +471,19 @@ class SimpleTextDocValuesReader extends
public long ramBytesUsed() {
return 0;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ BytesRef scratch = new BytesRef();
+ IndexInput clone = data.clone();
+ clone.seek(0);
+ ChecksumIndexInput input = new BufferedChecksumIndexInput(clone);
+ while(true) {
+ SimpleTextUtil.readLine(input, scratch);
+ if (scratch.equals(END)) {
+ SimpleTextUtil.checkFooter(input, CHECKSUM);
+ break;
+ }
+ }
+ }
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java Tue Apr 1 07:36:45 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
class SimpleTextDocValuesWriter extends DocValuesConsumer {
+ final static BytesRef CHECKSUM = new BytesRef("checksum ");
final static BytesRef END = new BytesRef("END");
final static BytesRef FIELD = new BytesRef("field ");
final static BytesRef TYPE = new BytesRef(" type ");
@@ -49,7 +50,7 @@ class SimpleTextDocValuesWriter extends
final static BytesRef NUMVALUES = new BytesRef(" numvalues ");
final static BytesRef ORDPATTERN = new BytesRef(" ordpattern ");
- final IndexOutput data;
+ IndexOutput data;
final BytesRef scratch = new BytesRef();
final int numDocs;
private final Set<String> fieldsSeen = new HashSet<>(); // for asserting
@@ -389,18 +390,25 @@ class SimpleTextDocValuesWriter extends
@Override
public void close() throws IOException {
- boolean success = false;
- try {
- assert !fieldsSeen.isEmpty();
- // TODO: sheisty to do this here?
- SimpleTextUtil.write(data, END);
- SimpleTextUtil.writeNewline(data);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(data);
- } else {
- IOUtils.closeWhileHandlingException(data);
+ if (data != null) {
+ boolean success = false;
+ try {
+ assert !fieldsSeen.isEmpty();
+ // TODO: sheisty to do this here?
+ SimpleTextUtil.write(data, END);
+ SimpleTextUtil.writeNewline(data);
+ String checksum = Long.toString(data.getChecksum());
+ SimpleTextUtil.write(data, CHECKSUM);
+ SimpleTextUtil.write(data, checksum, scratch);
+ SimpleTextUtil.writeNewline(data);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(data);
+ } else {
+ IOUtils.closeWhileHandlingException(data);
+ }
+ data = null;
}
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosReader.java Tue Apr 1 07:36:45 2014
@@ -24,15 +24,14 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.codecs.FieldInfosReader;
-import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
@@ -50,7 +49,7 @@ public class SimpleTextFieldInfosReader
@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext iocontext) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, FIELD_INFOS_EXTENSION);
- IndexInput input = directory.openInput(fileName, iocontext);
+ ChecksumIndexInput input = directory.openChecksumInput(fileName, iocontext);
BytesRef scratch = new BytesRef();
boolean success = false;
@@ -130,9 +129,7 @@ public class SimpleTextFieldInfosReader
infos[i].setDocValuesGen(dvGen);
}
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
- }
+ SimpleTextUtil.checkFooter(input, CHECKSUM);
FieldInfos fieldInfos = new FieldInfos(infos);
success = true;
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosWriter.java Tue Apr 1 07:36:45 2014
@@ -58,6 +58,7 @@ public class SimpleTextFieldInfosWriter
static final BytesRef NUM_ATTS = new BytesRef(" attributes ");
final static BytesRef ATT_KEY = new BytesRef(" key ");
final static BytesRef ATT_VALUE = new BytesRef(" value ");
+ final static BytesRef CHECKSUM = new BytesRef("checksum ");
@Override
public void write(Directory directory, String segmentName, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
@@ -132,6 +133,10 @@ public class SimpleTextFieldInfosWriter
}
}
}
+ String checksum = Long.toString(out.getChecksum());
+ SimpleTextUtil.write(out, CHECKSUM);
+ SimpleTextUtil.write(out, checksum, scratch);
+ SimpleTextUtil.writeNewline(out);
success = true;
} finally {
if (success) {
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Tue Apr 1 07:36:45 2014
@@ -34,6 +34,8 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.BufferedChecksumIndexInput;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
@@ -51,22 +53,23 @@ import org.apache.lucene.util.fst.PairOu
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.CHECKSUM;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.END;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.FIELD;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.TERM;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.DOC;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.FREQ;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.POS;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.START_OFFSET;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.END_OFFSET;
+import static org.apache.lucene.codecs.simpletext.SimpleTextFieldsWriter.PAYLOAD;
+
class SimpleTextFieldsReader extends FieldsProducer {
private final TreeMap<String,Long> fields;
private final IndexInput in;
private final FieldInfos fieldInfos;
private final int maxDoc;
- final static BytesRef END = SimpleTextFieldsWriter.END;
- final static BytesRef FIELD = SimpleTextFieldsWriter.FIELD;
- final static BytesRef TERM = SimpleTextFieldsWriter.TERM;
- final static BytesRef DOC = SimpleTextFieldsWriter.DOC;
- final static BytesRef FREQ = SimpleTextFieldsWriter.FREQ;
- final static BytesRef POS = SimpleTextFieldsWriter.POS;
- final static BytesRef START_OFFSET = SimpleTextFieldsWriter.START_OFFSET;
- final static BytesRef END_OFFSET = SimpleTextFieldsWriter.END_OFFSET;
- final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD;
-
public SimpleTextFieldsReader(SegmentReadState state) throws IOException {
this.maxDoc = state.segmentInfo.getDocCount();
fieldInfos = state.fieldInfos;
@@ -83,16 +86,18 @@ class SimpleTextFieldsReader extends Fie
}
private TreeMap<String,Long> readFields(IndexInput in) throws IOException {
+ ChecksumIndexInput input = new BufferedChecksumIndexInput(in);
BytesRef scratch = new BytesRef(10);
TreeMap<String,Long> fields = new TreeMap<>();
while (true) {
- SimpleTextUtil.readLine(in, scratch);
+ SimpleTextUtil.readLine(input, scratch);
if (scratch.equals(END)) {
+ SimpleTextUtil.checkFooter(input, CHECKSUM);
return fields;
} else if (StringHelper.startsWith(scratch, FIELD)) {
String fieldName = new String(scratch.bytes, scratch.offset + FIELD.length, scratch.length - FIELD.length, StandardCharsets.UTF_8);
- fields.put(fieldName, in.getFilePointer());
+ fields.put(fieldName, input.getFilePointer());
}
}
}
@@ -669,4 +674,7 @@ class SimpleTextFieldsReader extends Fie
}
return sizeInBytes;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java Tue Apr 1 07:36:45 2014
@@ -35,10 +35,11 @@ import org.apache.lucene.util.IOUtils;
class SimpleTextFieldsWriter extends FieldsConsumer implements Closeable {
- private final IndexOutput out;
+ private IndexOutput out;
private final BytesRef scratch = new BytesRef(10);
private final SegmentWriteState writeState;
+ final static BytesRef CHECKSUM = new BytesRef("checksum ");
final static BytesRef END = new BytesRef("END");
final static BytesRef FIELD = new BytesRef("field ");
final static BytesRef TERM = new BytesRef(" term ");
@@ -215,11 +216,18 @@ class SimpleTextFieldsWriter extends Fie
@Override
public void close() throws IOException {
- try {
- write(END);
- newline();
- } finally {
- out.close();
+ if (out != null) {
+ try {
+ write(END);
+ newline();
+ String checksum = Long.toString(out.getChecksum());
+ write(CHECKSUM);
+ write(checksum);
+ newline();
+ } finally {
+ out.close();
+ out = null;
+ }
}
}
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java Tue Apr 1 07:36:45 2014
@@ -24,9 +24,9 @@ import java.util.Collection;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
@@ -50,6 +50,7 @@ public class SimpleTextLiveDocsFormat ex
final static BytesRef SIZE = new BytesRef("size ");
final static BytesRef DOC = new BytesRef(" doc ");
final static BytesRef END = new BytesRef("END");
+ final static BytesRef CHECKSUM = new BytesRef("checksum ");
@Override
public MutableBits newLiveDocs(int size) throws IOException {
@@ -69,10 +70,10 @@ public class SimpleTextLiveDocsFormat ex
CharsRef scratchUTF16 = new CharsRef();
String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getDelGen());
- IndexInput in = null;
+ ChecksumIndexInput in = null;
boolean success = false;
try {
- in = dir.openInput(fileName, context);
+ in = dir.openChecksumInput(fileName, context);
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch, SIZE);
@@ -88,6 +89,8 @@ public class SimpleTextLiveDocsFormat ex
SimpleTextUtil.readLine(in, scratch);
}
+ SimpleTextUtil.checkFooter(in, CHECKSUM);
+
success = true;
return new SimpleTextBits(bits, size);
} finally {
@@ -127,6 +130,10 @@ public class SimpleTextLiveDocsFormat ex
SimpleTextUtil.write(out, END);
SimpleTextUtil.writeNewline(out);
+ String checksum = Long.toString(out.getChecksum());
+ SimpleTextUtil.write(out, CHECKSUM);
+ SimpleTextUtil.write(out, checksum, scratch);
+ SimpleTextUtil.writeNewline(out);
success = true;
} finally {
if (success) {
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoReader.java Tue Apr 1 07:36:45 2014
@@ -17,6 +17,7 @@ package org.apache.lucene.codecs.simplet
* limitations under the License.
*/
+import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_CHECKSUM;
import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_DIAG_KEY;
import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_DIAG_VALUE;
import static org.apache.lucene.codecs.simpletext.SimpleTextSegmentInfoWriter.SI_DOCCOUNT;
@@ -36,9 +37,9 @@ import java.util.Set;
import org.apache.lucene.codecs.SegmentInfoReader;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
@@ -55,7 +56,7 @@ public class SimpleTextSegmentInfoReader
public SegmentInfo read(Directory directory, String segmentName, IOContext context) throws IOException {
BytesRef scratch = new BytesRef();
String segFileName = IndexFileNames.segmentFileName(segmentName, "", SimpleTextSegmentInfoFormat.SI_EXTENSION);
- IndexInput input = directory.openInput(segFileName, context);
+ ChecksumIndexInput input = directory.openChecksumInput(segFileName, context);
boolean success = false;
try {
SimpleTextUtil.readLine(input, scratch);
@@ -97,6 +98,8 @@ public class SimpleTextSegmentInfoReader
String fileName = readString(SI_FILE.length, scratch);
files.add(fileName);
}
+
+ SimpleTextUtil.checkFooter(input, SI_CHECKSUM);
SegmentInfo info = new SegmentInfo(directory, version, segmentName, docCount,
isCompoundFile, null, diagnostics);
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoWriter.java Tue Apr 1 07:36:45 2014
@@ -47,6 +47,7 @@ public class SimpleTextSegmentInfoWriter
final static BytesRef SI_DIAG_VALUE = new BytesRef(" value ");
final static BytesRef SI_NUM_FILES = new BytesRef(" files ");
final static BytesRef SI_FILE = new BytesRef(" file ");
+ final static BytesRef SI_CHECKSUM = new BytesRef(" checksum ");
@Override
public void write(Directory dir, SegmentInfo si, FieldInfos fis, IOContext ioContext) throws IOException {
@@ -55,7 +56,7 @@ public class SimpleTextSegmentInfoWriter
si.addFile(segFileName);
boolean success = false;
- IndexOutput output = dir.createOutput(segFileName, ioContext);
+ IndexOutput output = dir.createOutput(segFileName, ioContext);
try {
BytesRef scratch = new BytesRef();
@@ -103,6 +104,11 @@ public class SimpleTextSegmentInfoWriter
SimpleTextUtil.writeNewline(output);
}
}
+
+ String checksum = Long.toString(output.getChecksum());
+ SimpleTextUtil.write(output, SI_CHECKSUM);
+ SimpleTextUtil.write(output, checksum, scratch);
+ SimpleTextUtil.writeNewline(output);
success = true;
} finally {
if (!success) {