You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/06/12 00:33:13 UTC
svn commit: r1349052 - in /lucene/dev/trunk: ./ lucene/ lucene/core/
lucene/core/src/java/org/apache/lucene/store/
lucene/core/src/test/org/apache/lucene/codecs/lucene40/
lucene/core/src/test/org/apache/lucene/index/
Author: rmuir
Date: Mon Jun 11 22:33:13 2012
New Revision: 1349052
URL: http://svn.apache.org/viewvc?rev=1349052&view=rev
Log:
LUCENE-4131: add codecheader to .cfs/.cfe
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/core/ (props changed)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java?rev=1349052&r1=1349051&r2=1349052&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java Mon Jun 11 22:33:13 2012
@@ -22,6 +22,7 @@ import org.apache.lucene.codecs.LiveDocs
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexFormatTooOldException;
import org.apache.lucene.store.DataOutput; // javadocs
+import org.apache.lucene.util.CodecUtil; // javadocs
import org.apache.lucene.util.IOUtils;
import java.util.Collection;
@@ -51,10 +52,10 @@ import java.io.IOException;
* </ul>
* <p>Description:</p>
* <ul>
- * <li>Compound (.cfs) --> FileData <sup>FileCount</sup></li>
- * <li>Compound Entry Table (.cfe) --> Version, FileCount, <FileName,
+ * <li>Compound (.cfs) --> Header, FileData <sup>FileCount</sup></li>
+ * <li>Compound Entry Table (.cfe) --> Header, FileCount, <FileName,
* DataOffset, DataLength> <sup>FileCount</sup></li>
- * <li>Version --> {@link DataOutput#writeInt Int32}</li>
+ * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>FileCount --> {@link DataOutput#writeVInt VInt}</li>
* <li>DataOffset,DataLength --> {@link DataOutput#writeLong UInt64}</li>
* <li>FileName --> {@link DataOutput#writeString String}</li>
@@ -114,22 +115,23 @@ public final class CompoundFileDirectory
/** Helper method that reads CFS entries from an input stream */
private static final Map<String, FileEntry> readEntries(
IndexInputSlicer handle, Directory dir, String name) throws IOException {
- // read the first VInt. If it is negative, it's the version number
- // otherwise it's the count (pre-3.1 indexes)
final IndexInput stream = handle.openFullSlice();
final Map<String, FileEntry> mapping;
boolean success = false;
try {
- final int firstInt = stream.readVInt();
- if (firstInt == CompoundFileWriter.FORMAT_CURRENT) {
+ final int firstInt = stream.readInt();
+ // NOTE: as long as we want to throw indexformattooold (vs corruptindexexception), we need
+ // to read the magic ourselves. See SegmentInfos which also has this.
+ if (firstInt == CodecUtil.CODEC_MAGIC) {
+ CodecUtil.checkHeaderNoMagic(stream, CompoundFileWriter.DATA_CODEC,
+ CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START);
IndexInput input = null;
try {
final String entriesFileName = IndexFileNames.segmentFileName(
IndexFileNames.stripExtension(name), "",
IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
input = dir.openInput(entriesFileName, IOContext.READONCE);
- final int readInt = input.readInt(); // unused right now
- assert readInt == CompoundFileWriter.ENTRY_FORMAT_CURRENT;
+ CodecUtil.checkHeader(input, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START);
final int numEntries = input.readVInt();
mapping = new HashMap<String, CompoundFileDirectory.FileEntry>(
numEntries);
@@ -147,8 +149,8 @@ public final class CompoundFileDirectory
IOUtils.close(input);
}
} else {
- throw new IndexFormatTooOldException(stream, firstInt,
- CompoundFileWriter.FORMAT_CURRENT, CompoundFileWriter.FORMAT_CURRENT);
+ throw new IndexFormatTooOldException(stream, firstInt,
+ CodecUtil.CODEC_MAGIC, CodecUtil.CODEC_MAGIC);
}
} finally {
if (success) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1349052&r1=1349051&r2=1349052&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java Mon Jun 11 22:33:13 2012
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.MergePolicy.MergeAbortedException;
+import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
/**
@@ -70,20 +71,13 @@ final class CompoundFileWriter implement
Directory dir;
}
- // Before versioning started.
- static final int FORMAT_PRE_VERSION = 0;
+ // versioning for the .cfs file
+ static final String DATA_CODEC = "CompoundFileWriterData";
+ static final int VERSION_START = 0;
+ static final int VERSION_CURRENT = VERSION_START;
- // Segment name is not written in the file names.
- static final int FORMAT_NO_SEGMENT_PREFIX = -1;
- static final int FORMAT_APPEND_FILES = -2;
-
- static final int ENTRY_FORMAT_CURRENT = -1;
-
- // NOTE: if you introduce a new format, make it 1 lower
- // than the current one, and always change this if you
- // switch to a new format!
- /** @lucene.internal */
- static final int FORMAT_CURRENT = FORMAT_APPEND_FILES;
+ // versioning for the .cfe file
+ static final String ENTRY_CODEC = "CompoundFileWriterEntries";
private final Directory directory;
private final Map<String, FileEntry> entries = new HashMap<String, FileEntry>();
@@ -121,7 +115,7 @@ final class CompoundFileWriter implement
boolean success = false;
try {
dataOut = directory.createOutput(dataFileName, IOContext.DEFAULT);
- dataOut.writeVInt(FORMAT_CURRENT);
+ CodecUtil.writeHeader(dataOut, DATA_CODEC, VERSION_CURRENT);
success = true;
} finally {
if (!success) {
@@ -228,7 +222,7 @@ final class CompoundFileWriter implement
protected void writeEntryTable(Collection<FileEntry> entries,
IndexOutput entryOut) throws IOException {
- entryOut.writeInt(ENTRY_FORMAT_CURRENT);
+ CodecUtil.writeHeader(entryOut, ENTRY_CODEC, VERSION_CURRENT);
entryOut.writeVInt(entries.size());
for (FileEntry fe : entries) {
entryOut.writeString(IndexFileNames.stripSegmentName(fe.file));
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java?rev=1349052&r1=1349051&r2=1349052&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java Mon Jun 11 22:33:13 2012
@@ -72,10 +72,6 @@ public class TestAllFilesHaveCodecHeader
CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
checkHeaders(cfsDir); // recurse into cfs
cfsDir.close();
- continue; // .cfs has its own header... would be nice to fix
- }
- if (file.endsWith(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)) {
- continue; // .cfe has its own header... would be nice to fix
}
IndexInput in = null;
boolean success = false;
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip?rev=1349052&r1=1349051&r2=1349052&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip?rev=1349052&r1=1349051&r2=1349052&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip?rev=1349052&r1=1349051&r2=1349052&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip?rev=1349052&r1=1349051&r2=1349052&view=diff
==============================================================================
Binary files - no diff available.