You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/04/01 09:36:48 UTC
svn commit: r1583550 [3/3] - in /lucene/dev/trunk: lucene/
lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/
lucene/codecs/src/java/org/apache/lucene/codecs/memory/
lucene/codecs/src/java...
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosReader.java Tue Apr 1 07:36:45 2014
@@ -29,6 +29,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -49,13 +50,13 @@ final class Lucene46FieldInfosReader ext
@Override
public FieldInfos read(Directory directory, String segmentName, String segmentSuffix, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segmentName, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
- IndexInput input = directory.openInput(fileName, context);
+ ChecksumIndexInput input = directory.openChecksumInput(fileName, context);
boolean success = false;
try {
- CodecUtil.checkHeader(input, Lucene46FieldInfosFormat.CODEC_NAME,
- Lucene46FieldInfosFormat.FORMAT_START,
- Lucene46FieldInfosFormat.FORMAT_CURRENT);
+ int codecVersion = CodecUtil.checkHeader(input, Lucene46FieldInfosFormat.CODEC_NAME,
+ Lucene46FieldInfosFormat.FORMAT_START,
+ Lucene46FieldInfosFormat.FORMAT_CURRENT);
final int size = input.readVInt(); //read in the size
FieldInfo infos[] = new FieldInfo[size];
@@ -91,9 +92,11 @@ final class Lucene46FieldInfosReader ext
omitNorms, storePayloads, indexOptions, docValuesType, normsType, Collections.unmodifiableMap(attributes));
infos[i].setDocValuesGen(dvGen);
}
-
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
+
+ if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM) {
+ CodecUtil.checkFooter(input);
+ } else {
+ CodecUtil.checkEOF(input);
}
FieldInfos fieldInfos = new FieldInfos(infos);
success = true;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosWriter.java Tue Apr 1 07:36:45 2014
@@ -26,9 +26,9 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
/**
@@ -81,6 +81,7 @@ final class Lucene46FieldInfosWriter ext
output.writeLong(fi.getDocValuesGen());
output.writeStringStringMap(fi.attributes());
}
+ CodecUtil.writeFooter(output);
success = true;
} finally {
if (success) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoFormat.java Tue Apr 1 07:36:45 2014
@@ -31,7 +31,7 @@ import org.apache.lucene.store.DataOutpu
* <p>
* Files:
* <ul>
- * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files
+ * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Footer
* </ul>
* </p>
* Data types:
@@ -43,6 +43,7 @@ import org.apache.lucene.store.DataOutpu
* <li>Files --> {@link DataOutput#writeStringSet Set<String>}</li>
* <li>Diagnostics --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* </p>
* Field Descriptions:
@@ -53,9 +54,6 @@ import org.apache.lucene.store.DataOutpu
* <li>IsCompoundFile records whether the segment is written as a compound file or
* not. If this is -1, the segment is not a compound file. If it is 1, the segment
* is a compound file.</li>
- * <li>Checksum contains the CRC32 checksum of all bytes in the segments_N file up
- * until the checksum. This is used to verify integrity of the file on opening the
- * index.</li>
* <li>The Diagnostics Map is privately written by {@link IndexWriter}, as a debugging aid,
* for each segment it creates. It includes metadata like the current Lucene
* version, OS, Java version, why the segment was created (merge, flush,
@@ -89,5 +87,6 @@ public class Lucene46SegmentInfoFormat e
public final static String SI_EXTENSION = "si";
static final String CODEC_NAME = "Lucene46SegmentInfo";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_CHECKSUM = 1;
+ static final int VERSION_CURRENT = VERSION_CHECKSUM;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoReader.java Tue Apr 1 07:36:45 2014
@@ -26,9 +26,9 @@ import org.apache.lucene.codecs.SegmentI
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.IOUtils;
/**
@@ -46,12 +46,12 @@ public class Lucene46SegmentInfoReader e
@Override
public SegmentInfo read(Directory dir, String segment, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene46SegmentInfoFormat.SI_EXTENSION);
- final IndexInput input = dir.openInput(fileName, context);
+ final ChecksumIndexInput input = dir.openChecksumInput(fileName, context);
boolean success = false;
try {
- CodecUtil.checkHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME,
- Lucene46SegmentInfoFormat.VERSION_START,
- Lucene46SegmentInfoFormat.VERSION_CURRENT);
+ int codecVersion = CodecUtil.checkHeader(input, Lucene46SegmentInfoFormat.CODEC_NAME,
+ Lucene46SegmentInfoFormat.VERSION_START,
+ Lucene46SegmentInfoFormat.VERSION_CURRENT);
final String version = input.readString();
final int docCount = input.readInt();
if (docCount < 0) {
@@ -61,8 +61,10 @@ public class Lucene46SegmentInfoReader e
final Map<String,String> diagnostics = input.readStringStringMap();
final Set<String> files = input.readStringSet();
- if (input.getFilePointer() != input.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + fileName + "\": read " + input.getFilePointer() + " vs size " + input.length() + " (resource: " + input + ")");
+ if (codecVersion >= Lucene46SegmentInfoFormat.VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(input);
+ } else {
+ CodecUtil.checkEOF(input);
}
final SegmentInfo si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java Tue Apr 1 07:36:45 2014
@@ -59,7 +59,7 @@ public class Lucene46SegmentInfoWriter e
output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO));
output.writeStringStringMap(si.getDiagnostics());
output.writeStringSet(si.files());
-
+ CodecUtil.writeFooter(output);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene46/package.html Tue Apr 1 07:36:45 2014
@@ -383,6 +383,9 @@ on multi-valued fields.</li>
<li>In version 4.5, DocValues were extended to explicitly represent missing values.</li>
<li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
allow updating NumericDocValues fields.</li>
+<li>In version 4.8, checksum footers were added to the end of each index file
+for improved data integrity. Specifically, the last 8 bytes of every index file
+contain the zlib-crc32 checksum of the file.</li>
</ul>
<a name="Limitations" id="Limitations"></a>
<h2>Limitations</h2>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Tue Apr 1 07:36:45 2014
@@ -310,6 +310,13 @@ public abstract class PerFieldDocValuesF
}
return size;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ for (DocValuesProducer format : formats.values()) {
+ format.checkIntegrity();
+ }
+ }
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Tue Apr 1 07:36:45 2014
@@ -246,6 +246,13 @@ public abstract class PerFieldPostingsFo
}
return sizeInBytes;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ for (FieldsProducer producer : formats.values()) {
+ producer.checkIntegrity();
+ }
+ }
}
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java Tue Apr 1 07:36:45 2014
@@ -238,4 +238,13 @@ public abstract class AtomicReader exten
* synchronization.
*/
public abstract Bits getLiveDocs();
+
+ /**
+ * Checks consistency of this reader.
+ * <p>
+ * Note that this may be costly in terms of I/O, e.g.
+ * may involve computing a checksum value against large data files.
+ * @lucene.internal
+ */
+ public abstract void checkIntegrity() throws IOException;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Tue Apr 1 07:36:45 2014
@@ -536,6 +536,10 @@ public class CheckIndex {
reader = new SegmentReader(info, IOContext.DEFAULT);
segInfoStat.openReaderPassed = true;
+
+ if (infoStream != null)
+ infoStream.print(" test: check integrity.........");
+ reader.checkIntegrity();
final int numDocs = reader.numDocs();
toLoseDocCount = numDocs;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Tue Apr 1 07:36:45 2014
@@ -423,4 +423,9 @@ public class FilterAtomicReader extends
return in.getDocsWithField(field);
}
+ @Override
+ public void checkIntegrity() throws IOException {
+ ensureOpen();
+ in.checkIntegrity();
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Apr 1 07:36:45 2014
@@ -2651,7 +2651,8 @@ public class IndexWriter implements Clos
false, codec, null);
SegmentMerger merger = new SegmentMerger(mergeReaders, info, infoStream, trackingDir,
- MergeState.CheckAbort.NONE, globalFieldNumberMap, context);
+ MergeState.CheckAbort.NONE, globalFieldNumberMap,
+ context, config.getCheckIntegrityAtMerge());
if (!merger.shouldMerge()) {
return;
@@ -4051,7 +4052,8 @@ public class IndexWriter implements Clos
// OneMerge to return a view over the actual segments to merge
final SegmentMerger merger = new SegmentMerger(merge.getMergeReaders(),
merge.info.info, infoStream, dirWrapper,
- checkAbort, globalFieldNumberMap, context);
+ checkAbort, globalFieldNumberMap,
+ context, config.getCheckIntegrityAtMerge());
merge.checkAborted(directory);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java Tue Apr 1 07:36:45 2014
@@ -110,6 +110,12 @@ public final class IndexWriterConfig ext
* (set to <code>true</code>). For batch indexing with very large
* ram buffers use <code>false</code> */
public final static boolean DEFAULT_USE_COMPOUND_FILE_SYSTEM = true;
+
+ /** Default value for calling {@link AtomicReader#checkIntegrity()} before
+ * merging segments (set to <code>false</code>). You can set this
+ * to <code>true</code> for additional safety. */
+ public final static boolean DEFAULT_CHECK_INTEGRITY_AT_MERGE = false;
+
/**
* Sets the default (for any instance) maximum time to wait for a write lock
* (in milliseconds).
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java Tue Apr 1 07:36:45 2014
@@ -97,6 +97,9 @@ public class LiveIndexWriterConfig {
/** True if segment flushes should use compound file format */
protected volatile boolean useCompoundFile = IndexWriterConfig.DEFAULT_USE_COMPOUND_FILE_SYSTEM;
+
+ /** True if merging should check integrity of segments before merge */
+ protected volatile boolean checkIntegrityAtMerge = IndexWriterConfig.DEFAULT_CHECK_INTEGRITY_AT_MERGE;
// used by IndexWriterConfig
LiveIndexWriterConfig(Analyzer analyzer, Version matchVersion) {
@@ -152,6 +155,7 @@ public class LiveIndexWriterConfig {
flushPolicy = config.getFlushPolicy();
perThreadHardLimitMB = config.getRAMPerThreadHardLimitMB();
useCompoundFile = config.getUseCompoundFile();
+ checkIntegrityAtMerge = config.getCheckIntegrityAtMerge();
}
/** Returns the default analyzer to use for indexing documents. */
@@ -475,6 +479,26 @@ public class LiveIndexWriterConfig {
return useCompoundFile ;
}
+ /**
+ * Sets if {@link IndexWriter} should call {@link AtomicReader#checkIntegrity()}
+ * on existing segments before merging them into a new one.
+ * <p>
+ * Use <code>true</code> to enable this safety check, which can help
+ * reduce the risk of propagating index corruption from older segments
+ * into new ones, at the expense of slower merging.
+ * </p>
+ */
+ public LiveIndexWriterConfig setCheckIntegrityAtMerge(boolean checkIntegrityAtMerge) {
+ this.checkIntegrityAtMerge = checkIntegrityAtMerge;
+ return this;
+ }
+
+ /** Returns true if {@link AtomicReader#checkIntegrity()} is called before
+ * merging segments. */
+ public boolean getCheckIntegrityAtMerge() {
+ return checkIntegrityAtMerge;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
@@ -499,6 +523,7 @@ public class LiveIndexWriterConfig {
sb.append("readerPooling=").append(getReaderPooling()).append("\n");
sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n");
sb.append("useCompoundFile=").append(getUseCompoundFile()).append("\n");
+ sb.append("checkIntegrityAtMerge=").append(getCheckIntegrityAtMerge()).append("\n");
return sb.toString();
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java Tue Apr 1 07:36:45 2014
@@ -299,4 +299,12 @@ public class ParallelAtomicReader extend
NumericDocValues values = reader == null ? null : reader.getNormValues(field);
return values;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ ensureOpen();
+ for (AtomicReader reader : completeReaderSet) {
+ reader.checkIntegrity();
+ }
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentDocValues.java Tue Apr 1 07:36:45 2014
@@ -95,14 +95,4 @@ final class SegmentDocValues {
IOUtils.reThrow(t);
}
}
-
- /** Returns approximate RAM bytes used. */
- synchronized long ramBytesUsed() {
- long ramBytesUsed = 0;
- for (RefCount<DocValuesProducer> dvp : genDVProducers.values()) {
- ramBytesUsed += dvp.get().ramBytesUsed();
- }
- return ramBytesUsed;
- }
-
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java Tue Apr 1 07:36:45 2014
@@ -36,11 +36,9 @@ import org.apache.lucene.codecs.CodecUti
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.store.ChecksumIndexInput;
-import org.apache.lucene.store.ChecksumIndexOutput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.IOUtils;
@@ -69,10 +67,10 @@ import org.apache.lucene.util.IOUtils;
* <p>
* Files:
* <ul>
- * <li><tt>segments.gen</tt>: GenHeader, Generation, Generation
+ * <li><tt>segments.gen</tt>: GenHeader, Generation, Generation, Footer
* <li><tt>segments_N</tt>: Header, Version, NameCounter, SegCount,
* <SegName, SegCodec, DelGen, DeletionCount, FieldInfosGen, UpdatesFiles><sup>SegCount</sup>,
- * CommitUserData, Checksum
+ * CommitUserData, Footer
* </ul>
* </p>
* Data types:
@@ -84,6 +82,7 @@ import org.apache.lucene.util.IOUtils;
* <li>SegName, SegCodec --> {@link DataOutput#writeString String}</li>
* <li>CommitUserData --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>UpdatesFiles --> {@link DataOutput#writeStringSet(Set) Set<String>}</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* </p>
* Field Descriptions:
@@ -98,9 +97,6 @@ import org.apache.lucene.util.IOUtils;
* there are no deletes. Anything above zero means there are deletes
* stored by {@link LiveDocsFormat}.</li>
* <li>DeletionCount records the number of deleted documents in this segment.</li>
- * <li>Checksum contains the CRC32 checksum of all bytes in the segments_N file up
- * until the checksum. This is used to verify integrity of the file on opening the
- * index.</li>
* <li>SegCodec is the {@link Codec#getName() name} of the Codec that encoded
* this segment.</li>
* <li>CommitUserData stores an optional user-supplied opaque
@@ -122,10 +118,17 @@ public final class SegmentInfos implemen
/** The file format version for the segments_N codec header, since 4.6+. */
public static final int VERSION_46 = 1;
+
+ /** The file format version for the segments_N codec header, since 4.8+ */
+ public static final int VERSION_48 = 2;
- /** Used for the segments.gen file only!
- * Whenever you add a new format, make it 1 smaller (negative version logic)! */
- public static final int FORMAT_SEGMENTS_GEN_CURRENT = -2;
+ // Used for the segments.gen file only!
+ // Whenever you add a new format, make it 1 smaller (negative version logic)!
+ private static final int FORMAT_SEGMENTS_GEN_47 = -2;
+ private static final int FORMAT_SEGMENTS_GEN_CHECKSUM = -3;
+ private static final int FORMAT_SEGMENTS_GEN_START = FORMAT_SEGMENTS_GEN_47;
+ /** Current format of segments.gen */
+ public static final int FORMAT_SEGMENTS_GEN_CURRENT = FORMAT_SEGMENTS_GEN_CHECKSUM;
/** Used to name new segments. */
public int counter;
@@ -266,6 +269,7 @@ public final class SegmentInfos implemen
genOutput.writeInt(FORMAT_SEGMENTS_GEN_CURRENT);
genOutput.writeLong(generation);
genOutput.writeLong(generation);
+ CodecUtil.writeFooter(genOutput);
} finally {
genOutput.close();
dir.sync(Collections.singleton(IndexFileNames.SEGMENTS_GEN));
@@ -317,7 +321,7 @@ public final class SegmentInfos implemen
lastGeneration = generation;
- ChecksumIndexInput input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ));
+ ChecksumIndexInput input = directory.openChecksumInput(segmentFileName, IOContext.READ);
try {
// NOTE: as long as we want to throw indexformattooold (vs corruptindexexception), we need
// to read the magic ourselves.
@@ -326,7 +330,7 @@ public final class SegmentInfos implemen
throw new IndexFormatTooOldException(input, magic, CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
}
// 4.0+
- int format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_46);
+ int format = CodecUtil.checkHeaderNoMagic(input, "segments", VERSION_40, VERSION_48);
version = input.readLong();
counter = input.readInt();
int numSegments = input.readInt();
@@ -366,10 +370,15 @@ public final class SegmentInfos implemen
}
userData = input.readStringStringMap();
- final long checksumNow = input.getChecksum();
- final long checksumThen = input.readLong();
- if (checksumNow != checksumThen) {
- throw new CorruptIndexException("checksum mismatch in segments file (resource: " + input + ")");
+ if (format >= VERSION_48) {
+ CodecUtil.checkFooter(input);
+ } else {
+ final long checksumNow = input.getChecksum();
+ final long checksumThen = input.readLong();
+ if (checksumNow != checksumThen) {
+ throw new CorruptIndexException("checksum mismatch in segments file (resource: " + input + ")");
+ }
+ CodecUtil.checkEOF(input);
}
success = true;
@@ -402,7 +411,7 @@ public final class SegmentInfos implemen
// Only non-null after prepareCommit has been called and
// before finishCommit is called
- ChecksumIndexOutput pendingSegnOutput;
+ IndexOutput pendingSegnOutput;
private void write(Directory directory) throws IOException {
@@ -415,12 +424,12 @@ public final class SegmentInfos implemen
generation++;
}
- ChecksumIndexOutput segnOutput = null;
+ IndexOutput segnOutput = null;
boolean success = false;
try {
- segnOutput = new ChecksumIndexOutput(directory.createOutput(segmentFileName, IOContext.DEFAULT));
- CodecUtil.writeHeader(segnOutput, "segments", VERSION_46);
+ segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
+ CodecUtil.writeHeader(segnOutput, "segments", VERSION_48);
segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
segnOutput.writeInt(size()); // write infos
@@ -641,9 +650,9 @@ public final class SegmentInfos implemen
// a stale cache (NFS) we have a better chance of
// getting the right generation.
long genB = -1;
- IndexInput genInput = null;
+ ChecksumIndexInput genInput = null;
try {
- genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
+ genInput = directory.openChecksumInput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE);
} catch (IOException e) {
if (infoStream != null) {
message("segments.gen open: IOException " + e);
@@ -653,18 +662,23 @@ public final class SegmentInfos implemen
if (genInput != null) {
try {
int version = genInput.readInt();
- if (version == FORMAT_SEGMENTS_GEN_CURRENT) {
+ if (version == FORMAT_SEGMENTS_GEN_47 || version == FORMAT_SEGMENTS_GEN_CHECKSUM) {
long gen0 = genInput.readLong();
long gen1 = genInput.readLong();
if (infoStream != null) {
message("fallback check: " + gen0 + "; " + gen1);
}
+ if (version == FORMAT_SEGMENTS_GEN_CHECKSUM) {
+ CodecUtil.checkFooter(genInput);
+ } else {
+ CodecUtil.checkEOF(genInput);
+ }
if (gen0 == gen1) {
// The file is consistent.
genB = gen0;
}
} else {
- throw new IndexFormatTooNewException(genInput, version, FORMAT_SEGMENTS_GEN_CURRENT, FORMAT_SEGMENTS_GEN_CURRENT);
+ throw new IndexFormatTooNewException(genInput, version, FORMAT_SEGMENTS_GEN_START, FORMAT_SEGMENTS_GEN_CURRENT);
}
} catch (IOException err2) {
// rethrow any format exception
@@ -863,7 +877,7 @@ public final class SegmentInfos implemen
}
boolean success = false;
try {
- pendingSegnOutput.finishCommit();
+ CodecUtil.writeFooter(pendingSegnOutput);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java Tue Apr 1 07:36:45 2014
@@ -52,7 +52,13 @@ final class SegmentMerger {
// note, just like in codec apis Directory 'dir' is NOT the same as segmentInfo.dir!!
SegmentMerger(List<AtomicReader> readers, SegmentInfo segmentInfo, InfoStream infoStream, Directory dir,
- MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context) throws IOException {
+ MergeState.CheckAbort checkAbort, FieldInfos.FieldNumbers fieldNumbers, IOContext context, boolean validate) throws IOException {
+ // validate incoming readers
+ if (validate) {
+ for (AtomicReader reader : readers) {
+ reader.checkIntegrity();
+ }
+ }
mergeState = new MergeState(readers, segmentInfo, infoStream, checkAbort);
directory = dir;
this.codec = segmentInfo.getCodec();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Tue Apr 1 07:36:45 2014
@@ -33,10 +33,13 @@ import org.apache.lucene.util.IOUtils;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
/**
* IndexReader implementation over a single segment.
@@ -72,7 +75,8 @@ public final class SegmentReader extends
}
};
- final Map<String,DocValuesProducer> dvProducers = new HashMap<>();
+ final Map<String,DocValuesProducer> dvProducersByField = new HashMap<>();
+ final Set<DocValuesProducer> dvProducers = Collections.newSetFromMap(new IdentityHashMap<DocValuesProducer,Boolean>());
final FieldInfos fieldInfos;
@@ -177,12 +181,15 @@ public final class SegmentReader extends
// System.out.println("[" + Thread.currentThread().getName() + "] SR.initDocValuesProducers: segInfo=" + si + "; gens=" + genInfos.keySet());
+ // TODO: can we avoid iterating over fieldinfos several times and creating maps of all this stuff if dv updates do not exist?
+
for (Entry<Long,List<FieldInfo>> e : genInfos.entrySet()) {
Long gen = e.getKey();
List<FieldInfo> infos = e.getValue();
DocValuesProducer dvp = segDocValues.getDocValuesProducer(gen, si, IOContext.READ, dir, dvFormat, infos);
for (FieldInfo fi : infos) {
- dvProducers.put(fi.name, dvp);
+ dvProducersByField.put(fi.name, dvp);
+ dvProducers.add(dvp);
}
}
@@ -250,7 +257,7 @@ public final class SegmentReader extends
try {
core.decRef();
} finally {
- dvProducers.clear();
+ dvProducersByField.clear();
try {
IOUtils.close(docValuesLocal, docsWithFieldLocal);
} finally {
@@ -395,13 +402,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Object> dvFields = docValuesLocal.get();
NumericDocValues dvs = (NumericDocValues) dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getNumeric(fi);
dvFields.put(field, dvs);
}
@@ -422,13 +428,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Bits> dvFields = docsWithFieldLocal.get();
Bits dvs = dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getDocsWithField(fi);
dvFields.put(field, dvs);
}
@@ -444,13 +449,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Object> dvFields = docValuesLocal.get();
BinaryDocValues dvs = (BinaryDocValues) dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getBinary(fi);
dvFields.put(field, dvs);
}
@@ -466,13 +470,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Object> dvFields = docValuesLocal.get();
SortedDocValues dvs = (SortedDocValues) dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getSorted(fi);
dvFields.put(field, dvs);
}
@@ -488,13 +491,12 @@ public final class SegmentReader extends
return null;
}
- DocValuesProducer dvProducer = dvProducers.get(field);
- assert dvProducer != null;
-
Map<String,Object> dvFields = docValuesLocal.get();
SortedSetDocValues dvs = (SortedSetDocValues) dvFields.get(field);
if (dvs == null) {
+ DocValuesProducer dvProducer = dvProducersByField.get(field);
+ assert dvProducer != null;
dvs = dvProducer.getSortedSet(fi);
dvFields.put(field, dvs);
}
@@ -548,12 +550,45 @@ public final class SegmentReader extends
public long ramBytesUsed() {
ensureOpen();
long ramBytesUsed = 0;
- if (segDocValues != null) {
- ramBytesUsed += segDocValues.ramBytesUsed();
+ if (dvProducers != null) {
+ for (DocValuesProducer producer : dvProducers) {
+ ramBytesUsed += producer.ramBytesUsed();
+ }
}
if (core != null) {
ramBytesUsed += core.ramBytesUsed();
}
return ramBytesUsed;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ ensureOpen();
+
+ // stored fields
+ getFieldsReader().checkIntegrity();
+
+ // term vectors
+ TermVectorsReader termVectorsReader = getTermVectorsReader();
+ if (termVectorsReader != null) {
+ termVectorsReader.checkIntegrity();
+ }
+
+ // terms/postings
+ if (core.fields != null) {
+ core.fields.checkIntegrity();
+ }
+
+ // norms
+ if (core.normsProducer != null) {
+ core.normsProducer.checkIntegrity();
+ }
+
+ // docvalues
+ if (dvProducers != null) {
+ for (DocValuesProducer producer : dvProducers) {
+ producer.checkIntegrity();
+ }
+ }
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Tue Apr 1 07:36:45 2014
@@ -239,4 +239,12 @@ public final class SlowCompositeReaderWr
// TODO: as this is a wrapper, should we really close the delegate?
in.close();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ ensureOpen();
+ for (AtomicReaderContext ctx : in.leaves()) {
+ ctx.reader().checkIntegrity();
+ }
+ }
}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java?rev=1583550&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksum.java Tue Apr 1 07:36:45 2014
@@ -0,0 +1,84 @@
+package org.apache.lucene.store;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.zip.Checksum;
+
+/**
+ * Wraps another {@link Checksum} with an internal buffer
+ * to speed up checksum calculations.
+ */
+public class BufferedChecksum implements Checksum {
+ private final Checksum in;
+ private final byte buffer[];
+ private int upto;
+ /** Default buffer size: 256 */
+ public static final int DEFAULT_BUFFERSIZE = 256;
+
+ /** Create a new BufferedChecksum with {@link #DEFAULT_BUFFERSIZE} */
+ public BufferedChecksum(Checksum in) {
+ this(in, DEFAULT_BUFFERSIZE);
+ }
+
+ /** Create a new BufferedChecksum with the specified bufferSize */
+ public BufferedChecksum(Checksum in, int bufferSize) {
+ this.in = in;
+ this.buffer = new byte[bufferSize];
+ }
+
+ @Override
+ public void update(int b) {
+ if (upto == buffer.length) {
+ flush();
+ }
+ buffer[upto++] = (byte) b;
+ }
+
+ @Override
+ public void update(byte[] b, int off, int len) {
+ if (len >= buffer.length) {
+ flush();
+ in.update(b, off, len);
+ } else {
+ if (upto + len > buffer.length) {
+ flush();
+ }
+ System.arraycopy(b, off, buffer, upto, len);
+ upto += len;
+ }
+ }
+
+ @Override
+ public long getValue() {
+ flush();
+ return in.getValue();
+ }
+
+ @Override
+ public void reset() {
+ upto = 0;
+ in.reset();
+ }
+
+ private void flush() {
+ if (upto > 0) {
+ in.update(buffer, 0, upto);
+ }
+ upto = 0;
+ }
+}
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksumIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksumIndexInput.java?rev=1583550&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksumIndexInput.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedChecksumIndexInput.java Tue Apr 1 07:36:45 2014
@@ -0,0 +1,72 @@
+package org.apache.lucene.store;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+/**
+ * Simple implementation of {@link ChecksumIndexInput} that wraps
+ * another input and delegates calls.
+ */
+public class BufferedChecksumIndexInput extends ChecksumIndexInput {
+ final IndexInput main;
+ final Checksum digest;
+
+ /** Creates a new BufferedChecksumIndexInput */
+ public BufferedChecksumIndexInput(IndexInput main) {
+ super("BufferedChecksumIndexInput(" + main + ")");
+ this.main = main;
+ this.digest = new BufferedChecksum(new CRC32());
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ final byte b = main.readByte();
+ digest.update(b);
+ return b;
+ }
+
+ @Override
+ public void readBytes(byte[] b, int offset, int len)
+ throws IOException {
+ main.readBytes(b, offset, len);
+ digest.update(b, offset, len);
+ }
+
+ @Override
+ public long getChecksum() {
+ return digest.getValue();
+ }
+
+ @Override
+ public void close() throws IOException {
+ main.close();
+ }
+
+ @Override
+ public long getFilePointer() {
+ return main.getFilePointer();
+ }
+
+ @Override
+ public long length() {
+ return main.length();
+ }
+}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/BufferedIndexOutput.java Tue Apr 1 07:36:45 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.zip.CRC32;
/** Base implementation class for buffered {@link IndexOutput}. */
public abstract class BufferedIndexOutput extends IndexOutput {
@@ -28,6 +29,7 @@ public abstract class BufferedIndexOutpu
private final byte[] buffer;
private long bufferStart = 0; // position in file of buffer
private int bufferPosition = 0; // position in buffer
+ private final CRC32 crc = new CRC32();
/**
* Creates a new {@link BufferedIndexOutput} with the default buffer size
@@ -75,6 +77,7 @@ public abstract class BufferedIndexOutpu
if (bufferPosition > 0)
flush();
// and write data at once
+ crc.update(b, offset, length);
flushBuffer(b, offset, length);
bufferStart += length;
} else {
@@ -99,6 +102,7 @@ public abstract class BufferedIndexOutpu
@Override
public void flush() throws IOException {
+ crc.update(buffer, 0, bufferPosition);
flushBuffer(buffer, bufferPosition);
bufferStart += bufferPosition;
bufferPosition = 0;
@@ -141,4 +145,9 @@ public abstract class BufferedIndexOutpu
return bufferSize;
}
+ @Override
+ public long getChecksum() throws IOException {
+ flush();
+ return crc.getValue();
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/ChecksumIndexInput.java Tue Apr 1 07:36:45 2014
@@ -1,5 +1,7 @@
package org.apache.lucene.store;
+import java.io.IOException;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,61 +19,24 @@ package org.apache.lucene.store;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.zip.CRC32;
-import java.util.zip.Checksum;
-
-/** Reads bytes through to a primary IndexInput, computing
- * checksum as it goes. Note that you cannot use seek().
- *
- * @lucene.internal
+/**
+ * Extension of IndexInput, computing checksum as it goes.
+ * Callers can retrieve the checksum via {@link #getChecksum()}.
*/
-public class ChecksumIndexInput extends IndexInput {
- IndexInput main;
- Checksum digest;
-
- public ChecksumIndexInput(IndexInput main) {
- super("ChecksumIndexInput(" + main + ")");
- this.main = main;
- digest = new CRC32();
- }
-
- @Override
- public byte readByte() throws IOException {
- final byte b = main.readByte();
- digest.update(b);
- return b;
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len)
- throws IOException {
- main.readBytes(b, offset, len);
- digest.update(b, offset, len);
- }
-
+public abstract class ChecksumIndexInput extends IndexInput {
- public long getChecksum() {
- return digest.getValue();
+ /** resourceDescription should be a non-null, opaque string
+ * describing this resource; it's returned from
+ * {@link #toString}. */
+ protected ChecksumIndexInput(String resourceDescription) {
+ super(resourceDescription);
}
- @Override
- public void close() throws IOException {
- main.close();
- }
-
- @Override
- public long getFilePointer() {
- return main.getFilePointer();
- }
+ /** Returns the current checksum value */
+ public abstract long getChecksum() throws IOException;
@Override
public void seek(long pos) {
throw new UnsupportedOperationException();
}
-
- @Override
- public long length() {
- return main.length();
- }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java Tue Apr 1 07:36:45 2014
@@ -52,14 +52,15 @@ import java.io.IOException;
* </ul>
* <p>Description:</p>
* <ul>
- * <li>Compound (.cfs) --> Header, FileData <sup>FileCount</sup></li>
+ * <li>Compound (.cfs) --> Header, FileData <sup>FileCount</sup>, Footer</li>
* <li>Compound Entry Table (.cfe) --> Header, FileCount, <FileName,
* DataOffset, DataLength> <sup>FileCount</sup></li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>FileCount --> {@link DataOutput#writeVInt VInt}</li>
- * <li>DataOffset,DataLength --> {@link DataOutput#writeLong UInt64}</li>
+ * <li>DataOffset,DataLength,Checksum --> {@link DataOutput#writeLong UInt64}</li>
* <li>FileName --> {@link DataOutput#writeString String}</li>
* <li>FileData --> raw file data</li>
+ * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
* <p>Notes:</p>
* <ul>
@@ -87,6 +88,7 @@ public final class CompoundFileDirectory
private static final Map<String,FileEntry> SENTINEL = Collections.emptyMap();
private final CompoundFileWriter writer;
private final IndexInputSlicer handle;
+ private int version;
/**
* Create a new CompoundFileDirectory.
@@ -120,15 +122,15 @@ public final class CompoundFileDirectory
}
/** Helper method that reads CFS entries from an input stream */
- private static final Map<String, FileEntry> readEntries(Directory dir, String name) throws IOException {
+ private final Map<String, FileEntry> readEntries(Directory dir, String name) throws IOException {
IOException priorE = null;
- IndexInput entriesStream = null;
+ ChecksumIndexInput entriesStream = null;
try {
final String entriesFileName = IndexFileNames.segmentFileName(
IndexFileNames.stripExtension(name), "",
IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
- entriesStream = dir.openInput(entriesFileName, IOContext.READONCE);
- CodecUtil.checkHeader(entriesStream, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START);
+ entriesStream = dir.openChecksumInput(entriesFileName, IOContext.READONCE);
+ version = CodecUtil.checkHeader(entriesStream, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_CURRENT);
final int numEntries = entriesStream.readVInt();
final Map<String, FileEntry> mapping = new HashMap<>(numEntries);
for (int i = 0; i < numEntries; i++) {
@@ -141,8 +143,10 @@ public final class CompoundFileDirectory
fileEntry.offset = entriesStream.readLong();
fileEntry.length = entriesStream.readLong();
}
- if (entriesStream.getFilePointer() != entriesStream.length()) {
- throw new CorruptIndexException("did not read all bytes from file \"" + entriesFileName + "\": read " + entriesStream.getFilePointer() + " vs size " + entriesStream.length() + " (resource: " + entriesStream + ")");
+ if (version >= CompoundFileWriter.VERSION_CHECKSUM) {
+ CodecUtil.checkFooter(entriesStream);
+ } else {
+ CodecUtil.checkEOF(entriesStream);
}
return mapping;
} catch (IOException ioe) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java Tue Apr 1 07:36:45 2014
@@ -54,7 +54,8 @@ final class CompoundFileWriter implement
// versioning for the .cfs file
static final String DATA_CODEC = "CompoundFileWriterData";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_CHECKSUM = 1;
+ static final int VERSION_CURRENT = VERSION_CHECKSUM;
// versioning for the .cfe file
static final String ENTRY_CODEC = "CompoundFileWriterEntries";
@@ -140,6 +141,7 @@ final class CompoundFileWriter implement
// open the compound stream
getOutput();
assert dataOut != null;
+ CodecUtil.writeFooter(dataOut);
} catch (IOException e) {
priorException = e;
} finally {
@@ -202,6 +204,7 @@ final class CompoundFileWriter implement
entryOut.writeLong(fe.offset);
entryOut.writeLong(fe.length);
}
+ CodecUtil.writeFooter(entryOut);
}
IndexOutput createOutput(String name, IOContext context) throws IOException {
@@ -342,6 +345,11 @@ final class CompoundFileWriter implement
writtenBytes += length;
delegate.writeBytes(b, offset, length);
}
+
+ @Override
+ public long getChecksum() throws IOException {
+ return delegate.getChecksum();
+ }
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/Directory.java Tue Apr 1 07:36:45 2014
@@ -100,7 +100,12 @@ public abstract class Directory implemen
* <p>Throws {@link FileNotFoundException} or {@link NoSuchFileException}
* if the file does not exist.
*/
- public abstract IndexInput openInput(String name, IOContext context) throws IOException;
+ public abstract IndexInput openInput(String name, IOContext context) throws IOException;
+
+ /** Returns a stream reading an existing file, computing checksum as it reads */
+ public ChecksumIndexInput openChecksumInput(String name, IOContext context) throws IOException {
+ return new BufferedChecksumIndexInput(openInput(name, context));
+ }
/** Construct a {@link Lock}.
* @param name the name of the lock file
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java Tue Apr 1 07:36:45 2014
@@ -43,6 +43,8 @@ public abstract class IndexOutput extend
*/
public abstract long getFilePointer();
+ /** Returns the current checksum of bytes written so far */
+ public abstract long getChecksum() throws IOException;
/** The number of bytes in the file. */
public abstract long length() throws IOException;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java Tue Apr 1 07:36:45 2014
@@ -18,6 +18,8 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
/**
* A memory-resident {@link IndexOutput} implementation.
@@ -35,6 +37,8 @@ public class RAMOutputStream extends Ind
private int bufferPosition;
private long bufferStart;
private int bufferLength;
+
+ private Checksum crc = new BufferedChecksum(new CRC32());
/** Construct an empty output buffer. */
public RAMOutputStream() {
@@ -95,6 +99,7 @@ public class RAMOutputStream extends Ind
bufferStart = 0;
bufferLength = 0;
file.setLength(0);
+ crc.reset();
}
@Override
@@ -113,12 +118,14 @@ public class RAMOutputStream extends Ind
currentBufferIndex++;
switchCurrentBuffer();
}
+ crc.update(b);
currentBuffer[bufferPosition++] = b;
}
@Override
public void writeBytes(byte[] b, int offset, int len) throws IOException {
assert b != null;
+ crc.update(b, offset, len);
while (len > 0) {
if (bufferPosition == bufferLength) {
currentBufferIndex++;
@@ -165,5 +172,10 @@ public class RAMOutputStream extends Ind
/** Returns byte usage of all buffers. */
public long sizeInBytes() {
return (long) file.numBuffers() * (long) BUFFER_SIZE;
- }
+ }
+
+ @Override
+ public long getChecksum() throws IOException {
+ return crc.getValue();
+ }
}
Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java?rev=1583550&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveChecksumFooter.java Tue Apr 1 07:36:45 2014
@@ -0,0 +1,90 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.lucene46.Lucene46Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.store.CompoundFileDirectory;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+/**
+ * Test that a plain default puts CRC32 footers in all files.
+ */
+public class TestAllFilesHaveChecksumFooter extends LuceneTestCase {
+ public void test() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ conf.setCodec(new Lucene46Codec());
+ RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
+ Document doc = new Document();
+ // these fields should sometimes get term vectors, etc
+ Field idField = newStringField("id", "", Field.Store.NO);
+ Field bodyField = newTextField("body", "", Field.Store.NO);
+ Field dvField = new NumericDocValuesField("dv", 5);
+ doc.add(idField);
+ doc.add(bodyField);
+ doc.add(dvField);
+ for (int i = 0; i < 100; i++) {
+ idField.setStringValue(Integer.toString(i));
+ bodyField.setStringValue(TestUtil.randomUnicodeString(random()));
+ riw.addDocument(doc);
+ if (random().nextInt(7) == 0) {
+ riw.commit();
+ }
+ if (random().nextInt(20) == 0) {
+ riw.deleteDocuments(new Term("id", Integer.toString(i)));
+ }
+ }
+ riw.close();
+ checkHeaders(dir);
+ dir.close();
+ }
+
+ private void checkHeaders(Directory dir) throws IOException {
+ for (String file : dir.listAll()) {
+ if (file.endsWith(IndexFileNames.COMPOUND_FILE_EXTENSION)) {
+ CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
+ checkHeaders(cfsDir); // recurse into cfs
+ cfsDir.close();
+ }
+ IndexInput in = null;
+ boolean success = false;
+ try {
+ in = dir.openInput(file, newIOContext(random()));
+ CodecUtil.checksumEntireFile(in);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(in);
+ } else {
+ IOUtils.closeWhileHandlingException(in);
+ }
+ }
+ }
+ }
+}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java Tue Apr 1 07:36:45 2014
@@ -24,6 +24,7 @@ import org.apache.lucene.codecs.CodecUti
import org.apache.lucene.codecs.lucene46.Lucene46Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
@@ -39,14 +40,15 @@ public class TestAllFilesHaveCodecHeader
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
conf.setCodec(new Lucene46Codec());
- // riw should sometimes create docvalues fields, etc
RandomIndexWriter riw = new RandomIndexWriter(random(), dir, conf);
Document doc = new Document();
// these fields should sometimes get term vectors, etc
Field idField = newStringField("id", "", Field.Store.NO);
Field bodyField = newTextField("body", "", Field.Store.NO);
+ Field dvField = new NumericDocValuesField("dv", 5);
doc.add(idField);
doc.add(bodyField);
+ doc.add(dvField);
for (int i = 0; i < 100; i++) {
idField.setStringValue(Integer.toString(i));
bodyField.setStringValue(TestUtil.randomUnicodeString(random()));
@@ -54,6 +56,10 @@ public class TestAllFilesHaveCodecHeader
if (random().nextInt(7) == 0) {
riw.commit();
}
+ // TODO: we should make a new format with a clean header...
+ // if (random().nextInt(20) == 0) {
+ // riw.deleteDocuments(new Term("id", Integer.toString(i)));
+ // }
}
riw.close();
checkHeaders(dir);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDoc.java Tue Apr 1 07:36:45 2014
@@ -222,7 +222,7 @@ public class TestDoc extends LuceneTestC
SegmentMerger merger = new SegmentMerger(Arrays.<AtomicReader>asList(r1, r2),
si, InfoStream.getDefault(), trackingDir,
- MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context);
+ MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), context, true);
MergeState mergeState = merger.merge();
r1.close();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue Apr 1 07:36:45 2014
@@ -83,7 +83,7 @@ public class TestSegmentMerger extends L
SegmentMerger merger = new SegmentMerger(Arrays.<AtomicReader>asList(reader1, reader2),
si, InfoStream.getDefault(), mergedDir,
- MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()));
+ MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()), true);
MergeState mergeState = merger.merge();
int docsMerged = mergeState.segmentInfo.getDocCount();
assertTrue(docsMerged == 2);
Added: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestBufferedChecksum.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestBufferedChecksum.java?rev=1583550&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestBufferedChecksum.java (added)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestBufferedChecksum.java Tue Apr 1 07:36:45 2014
@@ -0,0 +1,68 @@
+package org.apache.lucene.store;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.zip.CRC32;
+import java.util.zip.Checksum;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestBufferedChecksum extends LuceneTestCase {
+
+ public void testSimple() {
+ Checksum c = new BufferedChecksum(new CRC32());
+ c.update(1);
+ c.update(2);
+ c.update(3);
+ assertEquals(1438416925L, c.getValue());
+ }
+
+ public void testRandom() {
+ Checksum c1 = new CRC32();
+ Checksum c2 = new BufferedChecksum(new CRC32());
+ int iterations = atLeast(10000);
+ for (int i = 0; i < iterations; i++) {
+ switch(random().nextInt(4)) {
+ case 0:
+ // update(byte[], int, int)
+ int length = random().nextInt(1024);
+ byte bytes[] = new byte[length];
+ random().nextBytes(bytes);
+ c1.update(bytes, 0, bytes.length);
+ c2.update(bytes, 0, bytes.length);
+ break;
+ case 1:
+ // update(int)
+ int b = random().nextInt(256);
+ c1.update(b);
+ c2.update(b);
+ break;
+ case 2:
+ // reset()
+ c1.reset();
+ c2.reset();
+ break;
+ case 3:
+ // getValue()
+ assertEquals(c1.getValue(), c2.getValue());
+ break;
+ }
+ }
+ assertEquals(c1.getValue(), c2.getValue());
+ }
+}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestFilterDirectory.java Tue Apr 1 07:36:45 2014
@@ -30,12 +30,13 @@ public class TestFilterDirectory extends
public void testOverrides() throws Exception {
// verify that all methods of Directory are overridden by FilterDirectory,
// except those under the 'exclude' list
- Set<String> exclude = new HashSet<>();
- exclude.add("copy");
- exclude.add("createSlicer");
+ Set<Method> exclude = new HashSet<>();
+ exclude.add(Directory.class.getMethod("copy", Directory.class, String.class, String.class, IOContext.class));
+ exclude.add(Directory.class.getMethod("createSlicer", String.class, IOContext.class));
+ exclude.add(Directory.class.getMethod("openChecksumInput", String.class, IOContext.class));
for (Method m : FilterDirectory.class.getMethods()) {
if (m.getDeclaringClass() == Directory.class) {
- assertTrue("method " + m.getName() + " not overridden!", exclude.contains(m.getName()));
+ assertTrue("method " + m.getName() + " not overridden!", exclude.contains(m));
}
}
}
Modified: lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java (original)
+++ lucene/dev/trunk/lucene/facet/src/test/org/apache/lucene/facet/SlowRAMDirectory.java Tue Apr 1 07:36:45 2014
@@ -167,6 +167,7 @@ public class SlowRAMDirectory extends RA
@Override public void flush() throws IOException { io.flush(); }
@Override public long getFilePointer() { return io.getFilePointer(); }
@Override public long length() throws IOException { return io.length(); }
+ @Override public long getChecksum() throws IOException { return io.getChecksum(); }
}
}
Modified: lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Apr 1 07:36:45 2014
@@ -794,6 +794,11 @@ public class MemoryIndex {
return null;
}
+ @Override
+ public void checkIntegrity() throws IOException {
+ // no-op
+ }
+
private class MemoryFields extends Fields {
@Override
public Iterator<String> iterator() {
Modified: lucene/dev/trunk/lucene/misc/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/build.xml?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/build.xml (original)
+++ lucene/dev/trunk/lucene/misc/build.xml Tue Apr 1 07:36:45 2014
@@ -44,7 +44,7 @@
<target name="build-native-unix" depends="install-cpptasks">
<mkdir dir="${common.build.dir}/native"/>
- <cc outtype="shared" subsystem="console" outfile="${common.build.dir}/native/NativePosixUtil" >
+ <cc outtype="shared" name="c++" subsystem="console" outfile="${common.build.dir}/native/NativePosixUtil" >
<fileset file="${src.dir}/org/apache/lucene/store/NativePosixUtil.cpp" />
<includepath>
<pathelement location="${java.home}/../include"/>
@@ -54,7 +54,7 @@
</includepath>
<compilerarg value="-fPIC" />
- <linkerarg value="-lstdc++" />
+ <syslibset libs="stdc++"/>
</cc>
</target>
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/store/NativeUnixDirectory.java Tue Apr 1 07:36:45 2014
@@ -240,6 +240,11 @@ public class NativeUnixDirectory extends
public long length() {
return fileLength + bufferPos;
}
+
+ @Override
+ public long getChecksum() throws IOException {
+ throw new UnsupportedOperationException("this directory currently does not work at all!");
+ }
@Override
public void close() throws IOException {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java Tue Apr 1 07:36:45 2014
@@ -306,5 +306,10 @@ public class AssertingDocValuesFormat ex
public long ramBytesUsed() {
return in.ramBytesUsed();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
}
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java Tue Apr 1 07:36:45 2014
@@ -89,6 +89,11 @@ public final class AssertingPostingsForm
public long ramBytesUsed() {
return in.ramBytesUsed();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
}
static class AssertingFieldsConsumer extends FieldsConsumer {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java Tue Apr 1 07:36:45 2014
@@ -76,6 +76,11 @@ public class AssertingStoredFieldsFormat
public long ramBytesUsed() {
return in.ramBytesUsed();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
}
enum Status {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingTermVectorsFormat.java Tue Apr 1 07:36:45 2014
@@ -75,6 +75,11 @@ public class AssertingTermVectorsFormat
public long ramBytesUsed() {
return in.ramBytesUsed();
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ in.checkIntegrity();
+ }
}
enum Status {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java Tue Apr 1 07:36:45 2014
@@ -46,7 +46,7 @@ import org.apache.lucene.util.packed.Mon
import org.apache.lucene.util.packed.PackedInts.FormatAndBits;
import org.apache.lucene.util.packed.PackedInts;
-import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.VERSION_GCD_COMPRESSION;
import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.BLOCK_SIZE;
import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.BYTES;
import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesProducer.NUMBER;
@@ -71,10 +71,11 @@ class Lucene42DocValuesConsumer extends
try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
+ // this writer writes the format 4.2 did!
+ CodecUtil.writeHeader(data, dataCodec, VERSION_GCD_COMPRESSION);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
+ CodecUtil.writeHeader(meta, metaCodec, VERSION_GCD_COMPRESSION);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Tue Apr 1 07:36:45 2014
@@ -94,6 +94,9 @@ public final class RAMOnlyPostingsFormat
}
return sizeInBytes;
}
+
+ @Override
+ public void checkIntegrity() throws IOException {}
}
static class RAMField extends Terms {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/store/MockIndexOutputWrapper.java Tue Apr 1 07:36:45 2014
@@ -166,6 +166,11 @@ public class MockIndexOutputWrapper exte
}
@Override
+ public long getChecksum() throws IOException {
+ return delegate.getChecksum();
+ }
+
+ @Override
public String toString() {
return "MockIndexOutputWrapper(" + delegate + ")";
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Tue Apr 1 07:36:45 2014
@@ -810,6 +810,7 @@ public abstract class LuceneTestCase ext
}
c.setUseCompoundFile(r.nextBoolean());
c.setReaderPooling(r.nextBoolean());
+ c.setCheckIntegrityAtMerge(r.nextBoolean());
return c;
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java Tue Apr 1 07:36:45 2014
@@ -243,6 +243,7 @@ public final class TestUtil {
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
PrintStream infoStream = new PrintStream(bos, false, IOUtils.UTF_8);
+ reader.checkIntegrity();
FieldNormStatus fieldNormStatus = CheckIndex.testFieldNorms(reader, infoStream);
TermIndexStatus termIndexStatus = CheckIndex.testPostings(reader, infoStream);
StoredFieldStatus storedFieldStatus = CheckIndex.testStoredFields(reader, infoStream);
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/ThrottledIndexOutput.java Tue Apr 1 07:36:45 2014
@@ -145,4 +145,9 @@ public class ThrottledIndexOutput extend
public void copyBytes(DataInput input, long numBytes) throws IOException {
delegate.copyBytes(input, numBytes);
}
+
+ @Override
+ public long getChecksum() throws IOException {
+ return delegate.getChecksum();
+ }
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/blockcache/CachedIndexOutput.java Tue Apr 1 07:36:45 2014
@@ -88,5 +88,10 @@ public class CachedIndexOutput extends R
offset += len;
}
}
-
+
+ @Override
+ public long getChecksum() throws IOException {
+ flush();
+ return dest.getChecksum();
+ }
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/store/hdfs/NullIndexOutput.java Tue Apr 1 07:36:45 2014
@@ -66,5 +66,9 @@ public class NullIndexOutput extends Ind
length = pos;
}
}
-
+
+ @Override
+ public long getChecksum() throws IOException {
+ return 0; // we don't write anything.
+ }
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestDocSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestDocSet.java?rev=1583550&r1=1583549&r2=1583550&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestDocSet.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestDocSet.java Tue Apr 1 07:36:45 2014
@@ -418,6 +418,10 @@ public class TestDocSet extends LuceneTe
@Override
public void document(int doc, StoredFieldVisitor visitor) {
}
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ }
};
}