You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/06/12 00:06:46 UTC
svn commit: r1349048 - in /lucene/dev/branches/branch_4x/lucene/core/src:
java/org/apache/lucene/store/CompoundFileDirectory.java
java/org/apache/lucene/store/CompoundFileWriter.java
test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java
Author: rmuir
Date: Mon Jun 11 22:06:46 2012
New Revision: 1349048
URL: http://svn.apache.org/viewvc?rev=1349048&view=rev
Log:
LUCENE-4131: add codecheader to .cfs/.cfe
Modified:
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java?rev=1349048&r1=1349047&r2=1349048&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileDirectory.java Mon Jun 11 22:06:46 2012
@@ -22,6 +22,7 @@ import org.apache.lucene.codecs.LiveDocs
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.store.DataOutput; // javadocs
+import org.apache.lucene.util.CodecUtil; // javadocs
import org.apache.lucene.util.IOUtils;
import java.util.Collection;
@@ -51,10 +52,10 @@ import java.io.IOException;
* </ul>
* <p>Description:</p>
* <ul>
- * <li>Compound (.cfs) --> FileData <sup>FileCount</sup></li>
- * <li>Compound Entry Table (.cfe) --> Version, FileCount, <FileName,
+ * <li>Compound (.cfs) --> Header, FileData <sup>FileCount</sup></li>
+ * <li>Compound Entry Table (.cfe) --> Header, FileCount, <FileName,
* DataOffset, DataLength> <sup>FileCount</sup></li>
- * <li>Version --> {@link DataOutput#writeInt Int32}</li>
+ * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>FileCount --> {@link DataOutput#writeVInt VInt}</li>
* <li>DataOffset,DataLength --> {@link DataOutput#writeLong UInt64}</li>
* <li>FileName --> {@link DataOutput#writeString String}</li>
@@ -111,6 +112,11 @@ public final class CompoundFileDirectory
}
}
+ private static final byte CODEC_MAGIC_BYTE1 = (byte) (CodecUtil.CODEC_MAGIC >>> 24);
+ private static final byte CODEC_MAGIC_BYTE2 = (byte) (CodecUtil.CODEC_MAGIC >>> 16);
+ private static final byte CODEC_MAGIC_BYTE3 = (byte) (CodecUtil.CODEC_MAGIC >>> 8);
+ private static final byte CODEC_MAGIC_BYTE4 = (byte) CodecUtil.CODEC_MAGIC;
+
/** Helper method that reads CFS entries from an input stream */
private static final Map<String, FileEntry> readEntries(
IndexInputSlicer handle, Directory dir, String name) throws IOException {
@@ -121,15 +127,27 @@ public final class CompoundFileDirectory
boolean success = false;
try {
final int firstInt = stream.readVInt();
- if (firstInt == CompoundFileWriter.FORMAT_CURRENT) {
+ // impossible for 3.0 to have 63 files in a .cfs, CFS writer was not visible
+ // and separate norms/etc are outside of cfs.
+ if (firstInt == CODEC_MAGIC_BYTE1) {
+ byte secondByte = stream.readByte();
+ byte thirdByte = stream.readByte();
+ byte fourthByte = stream.readByte();
+ if (secondByte != CODEC_MAGIC_BYTE2 ||
+ thirdByte != CODEC_MAGIC_BYTE3 ||
+ fourthByte != CODEC_MAGIC_BYTE4) {
+ throw new CorruptIndexException("Illegal/impossible header for CFS file: "
+ + secondByte + "," + thirdByte + "," + fourthByte);
+ }
+ CodecUtil.checkHeaderNoMagic(stream, CompoundFileWriter.DATA_CODEC,
+ CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START);
IndexInput input = null;
try {
final String entriesFileName = IndexFileNames.segmentFileName(
IndexFileNames.stripExtension(name), "",
IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
input = dir.openInput(entriesFileName, IOContext.READONCE);
- final int readInt = input.readInt(); // unused right now
- assert readInt == CompoundFileWriter.ENTRY_FORMAT_CURRENT;
+ CodecUtil.checkHeader(input, CompoundFileWriter.ENTRY_CODEC, CompoundFileWriter.VERSION_START, CompoundFileWriter.VERSION_START);
final int numEntries = input.readVInt();
mapping = new HashMap<String, CompoundFileDirectory.FileEntry>(
numEntries);
@@ -166,9 +184,9 @@ public final class CompoundFileDirectory
final int count;
final boolean stripSegmentName;
if (firstInt < CompoundFileWriter.FORMAT_PRE_VERSION) {
- if (firstInt < CompoundFileWriter.FORMAT_CURRENT) {
+ if (firstInt < CompoundFileWriter.FORMAT_NO_SEGMENT_PREFIX) {
throw new CorruptIndexException("Incompatible format version: "
- + firstInt + " expected " + CompoundFileWriter.FORMAT_CURRENT + " (resource: " + stream + ")");
+ + firstInt + " expected >= " + CompoundFileWriter.FORMAT_NO_SEGMENT_PREFIX + " (resource: " + stream + ")");
}
// It's a post-3.1 index, read the count.
count = stream.readVInt();
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1349048&r1=1349047&r2=1349048&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/CompoundFileWriter.java Mon Jun 11 22:06:46 2012
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.MergePolicy.MergeAbortedException;
+import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
/**
@@ -75,15 +76,14 @@ final class CompoundFileWriter implement
// Segment name is not written in the file names.
static final int FORMAT_NO_SEGMENT_PREFIX = -1;
- static final int FORMAT_APPEND_FILES = -2;
-
- static final int ENTRY_FORMAT_CURRENT = -1;
+
+ // versioning for the .cfs file
+ static final String DATA_CODEC = "CompoundFileWriterData";
+ static final int VERSION_START = 0;
+ static final int VERSION_CURRENT = VERSION_START;
- // NOTE: if you introduce a new format, make it 1 lower
- // than the current one, and always change this if you
- // switch to a new format!
- /** @lucene.internal */
- static final int FORMAT_CURRENT = FORMAT_APPEND_FILES;
+ // versioning for the .cfe file
+ static final String ENTRY_CODEC = "CompoundFileWriterEntries";
private final Directory directory;
private final Map<String, FileEntry> entries = new HashMap<String, FileEntry>();
@@ -121,7 +121,7 @@ final class CompoundFileWriter implement
boolean success = false;
try {
dataOut = directory.createOutput(dataFileName, IOContext.DEFAULT);
- dataOut.writeVInt(FORMAT_CURRENT);
+ CodecUtil.writeHeader(dataOut, DATA_CODEC, VERSION_CURRENT);
success = true;
} finally {
if (!success) {
@@ -228,7 +228,7 @@ final class CompoundFileWriter implement
protected void writeEntryTable(Collection<FileEntry> entries,
IndexOutput entryOut) throws IOException {
- entryOut.writeInt(ENTRY_FORMAT_CURRENT);
+ CodecUtil.writeHeader(entryOut, ENTRY_CODEC, VERSION_CURRENT);
entryOut.writeVInt(entries.size());
for (FileEntry fe : entries) {
entryOut.writeString(IndexFileNames.stripSegmentName(fe.file));
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java?rev=1349048&r1=1349047&r2=1349048&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/lucene40/TestAllFilesHaveCodecHeader.java Mon Jun 11 22:06:46 2012
@@ -72,10 +72,6 @@ public class TestAllFilesHaveCodecHeader
CompoundFileDirectory cfsDir = new CompoundFileDirectory(dir, file, newIOContext(random()), false);
checkHeaders(cfsDir); // recurse into cfs
cfsDir.close();
- continue; // .cfs has its own header... would be nice to fix
- }
- if (file.endsWith(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)) {
- continue; // .cfe has its own header... would be nice to fix
}
IndexInput in = null;
boolean success = false;