You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/21 23:40:03 UTC
svn commit: r1633471 - in /lucene/dev/branches/lucene5969/lucene/core/src:
java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
Author: rmuir
Date: Tue Oct 21 21:40:02 2014
New Revision: 1633471
URL: http://svn.apache.org/r1633471
Log:
LUCENE-5969: test that default codec uses segmentheader for all files. change .si to write its ID the same way for consistency
Modified:
lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
Modified: lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java?rev=1633471&r1=1633470&r2=1633471&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SegmentInfoFormat.java Tue Oct 21 21:40:02 2014
@@ -42,20 +42,19 @@ import org.apache.lucene.util.Version;
* <p>
* Files:
* <ul>
- * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Id, Footer
+ * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Files, Footer
* </ul>
* </p>
* Data types:
* <p>
* <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
* <li>SegSize --> {@link DataOutput#writeInt Int32}</li>
* <li>SegVersion --> {@link DataOutput#writeString String}</li>
* <li>Files --> {@link DataOutput#writeStringSet Set<String>}</li>
* <li>Diagnostics --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
* <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}</li>
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
- * <li>Id --> {@link DataOutput#writeString String}</li>
* </ul>
* </p>
* Field Descriptions:
@@ -93,6 +92,12 @@ public class Lucene50SegmentInfoFormat e
CodecUtil.checkHeader(input, Lucene50SegmentInfoFormat.CODEC_NAME,
Lucene50SegmentInfoFormat.VERSION_START,
Lucene50SegmentInfoFormat.VERSION_CURRENT);
+ byte id[] = new byte[StringHelper.ID_LENGTH];
+ input.readBytes(id, 0, id.length);
+ String suffix = input.readString();
+ if (!suffix.isEmpty()) {
+ throw new CorruptIndexException("invalid codec header: got unexpected suffix: " + suffix, input);
+ }
final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
final int docCount = input.readInt();
@@ -103,9 +108,6 @@ public class Lucene50SegmentInfoFormat e
final Map<String,String> diagnostics = input.readStringStringMap();
final Set<String> files = input.readStringSet();
- byte[] id = new byte[StringHelper.ID_LENGTH];
- input.readBytes(id, 0, id.length);
-
si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, id);
si.setFiles(files);
} catch (Throwable exception) {
@@ -124,7 +126,12 @@ public class Lucene50SegmentInfoFormat e
boolean success = false;
try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
- CodecUtil.writeHeader(output, Lucene50SegmentInfoFormat.CODEC_NAME, Lucene50SegmentInfoFormat.VERSION_CURRENT);
+ // NOTE: we encode ID in the segment header, for format consistency with all other per-segment files
+ CodecUtil.writeSegmentHeader(output,
+ Lucene50SegmentInfoFormat.CODEC_NAME,
+ Lucene50SegmentInfoFormat.VERSION_CURRENT,
+ si.getId(),
+ "");
Version version = si.getVersion();
if (version.major < 5) {
throw new IllegalArgumentException("invalid major version: should be >= 5 but got: " + version.major + " segment=" + si);
@@ -145,11 +152,6 @@ public class Lucene50SegmentInfoFormat e
}
}
output.writeStringSet(files);
- byte[] id = si.getId();
- if (id.length != StringHelper.ID_LENGTH) {
- throw new IllegalArgumentException("invalid id, got=" + StringHelper.idToString(id));
- }
- output.writeBytes(id, 0, id.length);
CodecUtil.writeFooter(output);
success = true;
} finally {
Modified: lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java?rev=1633471&r1=1633470&r2=1633471&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/core/src/test/org/apache/lucene/index/TestAllFilesHaveCodecHeader.java Tue Oct 21 21:40:02 2014
@@ -32,10 +32,11 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.TestUtil;
/**
- * Test that a plain default puts codec headers in all files.
+ * Test that a plain default puts codec headers in all files
*/
public class TestAllFilesHaveCodecHeader extends LuceneTestCase {
public void test() throws Exception {
@@ -83,23 +84,24 @@ public class TestAllFilesHaveCodecHeader
private void checkHeaders(Directory dir, Map<String,String> namesToExtensions) throws IOException {
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
- checkHeader(dir, sis.getSegmentsFileName(), namesToExtensions);
+ checkHeader(dir, sis.getSegmentsFileName(), namesToExtensions, null);
for (SegmentCommitInfo si : sis) {
+ assertNotNull(si.info.getId());
for (String file : si.files()) {
- checkHeader(dir, file, namesToExtensions);
+ checkHeader(dir, file, namesToExtensions, si.info.getId());
}
if (si.info.getUseCompoundFile()) {
try (Directory cfsDir = si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info, newIOContext(random()))) {
for (String cfsFile : cfsDir.listAll()) {
- checkHeader(cfsDir, cfsFile, namesToExtensions);
+ checkHeader(cfsDir, cfsFile, namesToExtensions, si.info.getId());
}
}
}
}
}
- private void checkHeader(Directory dir, String file, Map<String,String> namesToExtensions) throws IOException {
+ private void checkHeader(Directory dir, String file, Map<String,String> namesToExtensions, byte[] id) throws IOException {
try (IndexInput in = dir.openInput(file, newIOContext(random()))) {
int val = in.readInt();
assertEquals(file + " has no codec header, instead found: " + val, CodecUtil.CODEC_MAGIC, val);
@@ -114,6 +116,15 @@ public class TestAllFilesHaveCodecHeader
if (previous != null && !previous.equals(extension)) {
fail("extensions " + previous + " and " + extension + " share same codecName " + codecName);
}
+ // read version
+ in.readInt();
+ // read segment id (except for segments_N)
+ if (id != null) {
+ byte actualID[] = new byte[StringHelper.ID_LENGTH];
+ in.readBytes(actualID, 0, actualID.length);
+ assertArrayEquals("expected " + StringHelper.idToString(id) + ", got " + StringHelper.idToString(actualID), id, actualID);
+ }
+
}
}
}