You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/01 15:14:41 UTC
svn commit: r1628697 - in /lucene/dev/branches/lucene5969/lucene:
codecs/src/java/org/apache/lucene/codecs/memory/
sandbox/src/java/org/apache/lucene/codecs/idversion/
Author: rmuir
Date: Wed Oct 1 13:14:41 2014
New Revision: 1628697
URL: http://svn.apache.org/r1628697
Log:
LUCENE-5969: give remaining codecs segment headers and cleanups
Modified:
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java
lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java
lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java
lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java Wed Oct 1 13:14:41 2014
@@ -75,7 +75,6 @@ public class FSTOrdTermsReader extends F
static final int INTERVAL = FSTOrdTermsWriter.SKIP_INTERVAL;
final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader;
- int version;
//static final boolean TEST = false;
public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
@@ -89,11 +88,20 @@ public class FSTOrdTermsReader extends F
try {
indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context);
blockIn = state.directory.openInput(termsBlockFileName, state.context);
- version = readHeader(indexIn);
- readHeader(blockIn);
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(blockIn);
+ int version = CodecUtil.checkSegmentHeader(indexIn, FSTOrdTermsWriter.TERMS_INDEX_CODEC_NAME,
+ FSTOrdTermsWriter.VERSION_START,
+ FSTOrdTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+ int version2 = CodecUtil.checkSegmentHeader(blockIn, FSTOrdTermsWriter.TERMS_CODEC_NAME,
+ FSTOrdTermsWriter.VERSION_START,
+ FSTOrdTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
+ if (version != version2) {
+ throw new CorruptIndexException("Format versions mismatch: index=" + version + ", terms=" + version2, blockIn);
}
+
+ CodecUtil.checksumEntireFile(blockIn);
this.postingsReader.init(blockIn);
seekDir(blockIn);
@@ -114,11 +122,7 @@ public class FSTOrdTermsReader extends F
TermsReader previous = fields.put(fieldInfo.name, current);
checkFieldSummary(state.segmentInfo, indexIn, blockIn, current, previous);
}
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checkFooter(indexIn);
- } else {
- CodecUtil.checkEOF(indexIn);
- }
+ CodecUtil.checkFooter(indexIn);
success = true;
} finally {
if (success) {
@@ -129,17 +133,8 @@ public class FSTOrdTermsReader extends F
}
}
- private int readHeader(IndexInput in) throws IOException {
- return CodecUtil.checkHeader(in, FSTOrdTermsWriter.TERMS_CODEC_NAME,
- FSTOrdTermsWriter.TERMS_VERSION_START,
- FSTOrdTermsWriter.TERMS_VERSION_CURRENT);
- }
private void seekDir(IndexInput in) throws IOException {
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- in.seek(in.length() - CodecUtil.footerLength() - 8);
- } else {
- in.seek(in.length() - 8);
- }
+ in.seek(in.length() - CodecUtil.footerLength() - 8);
in.seek(in.readLong());
}
private void checkFieldSummary(SegmentInfo info, IndexInput indexIn, IndexInput blockIn, TermsReader field, TermsReader previous) throws IOException {
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java Wed Oct 1 13:14:41 2014
@@ -39,7 +39,6 @@ import org.apache.lucene.store.RAMOutput
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
@@ -149,10 +148,11 @@ import org.apache.lucene.util.fst.Util;
public class FSTOrdTermsWriter extends FieldsConsumer {
static final String TERMS_INDEX_EXTENSION = "tix";
static final String TERMS_BLOCK_EXTENSION = "tbk";
- static final String TERMS_CODEC_NAME = "FST_ORD_TERMS_DICT";
- public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CHECKSUM = 1;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
+ static final String TERMS_CODEC_NAME = "FSTOrdTerms";
+ static final String TERMS_INDEX_CODEC_NAME = "FSTOrdIndex";
+
+ public static final int VERSION_START = 2;
+ public static final int VERSION_CURRENT = VERSION_START;
public static final int SKIP_INTERVAL = 8;
final PostingsWriterBase postingsWriter;
@@ -174,8 +174,10 @@ public class FSTOrdTermsWriter extends F
try {
this.indexOut = state.directory.createOutput(termsIndexFileName, state.context);
this.blockOut = state.directory.createOutput(termsBlockFileName, state.context);
- writeHeader(indexOut);
- writeHeader(blockOut);
+ CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.writeSegmentHeader(blockOut, TERMS_CODEC_NAME, VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter.init(blockOut);
success = true;
} finally {
@@ -260,9 +262,6 @@ public class FSTOrdTermsWriter extends F
}
}
- private void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
- }
private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
out.writeLong(dirStart);
}
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java Wed Oct 1 13:14:41 2014
@@ -72,7 +72,6 @@ public class FSTTermsReader extends Fiel
final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader;
//static boolean TEST = false;
- final int version;
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
@@ -82,10 +81,11 @@ public class FSTTermsReader extends Fiel
boolean success = false;
try {
- version = readHeader(in);
- if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(in);
- }
+ CodecUtil.checkSegmentHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
+ FSTTermsWriter.TERMS_VERSION_START,
+ FSTTermsWriter.TERMS_VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.checksumEntireFile(in);
this.postingsReader.init(in);
seekDir(in);
@@ -113,17 +113,8 @@ public class FSTTermsReader extends Fiel
}
}
- private int readHeader(IndexInput in) throws IOException {
- return CodecUtil.checkHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
- FSTTermsWriter.TERMS_VERSION_START,
- FSTTermsWriter.TERMS_VERSION_CURRENT);
- }
private void seekDir(IndexInput in) throws IOException {
- if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
- in.seek(in.length() - CodecUtil.footerLength() - 8);
- } else {
- in.seek(in.length() - 8);
- }
+ in.seek(in.length() - CodecUtil.footerLength() - 8);
in.seek(in.readLong());
}
private void checkFieldSummary(SegmentInfo info, IndexInput in, TermsReader field, TermsReader previous) throws IOException {
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java Wed Oct 1 13:14:41 2014
@@ -39,7 +39,6 @@ import org.apache.lucene.store.RAMOutput
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
@@ -123,10 +122,9 @@ import org.apache.lucene.util.fst.Util;
public class FSTTermsWriter extends FieldsConsumer {
static final String TERMS_EXTENSION = "tmp";
- static final String TERMS_CODEC_NAME = "FST_TERMS_DICT";
- public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CHECKSUM = 1;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
+ static final String TERMS_CODEC_NAME = "FSTTerms";
+ public static final int TERMS_VERSION_START = 2;
+ public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START;
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
@@ -144,7 +142,9 @@ public class FSTTermsWriter extends Fiel
boolean success = false;
try {
- writeHeader(out);
+ CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
this.postingsWriter.init(out);
success = true;
} finally {
@@ -154,10 +154,6 @@ public class FSTTermsWriter extends Fiel
}
}
- private void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
- }
-
private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
out.writeLong(dirStart);
}
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java Wed Oct 1 13:14:41 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.MathUtil;
import org.apache.lucene.util.fst.Builder;
@@ -75,10 +74,10 @@ class MemoryDocValuesConsumer extends Do
try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java Wed Oct 1 13:14:41 2014
@@ -110,7 +110,7 @@ class MemoryDocValuesProducer extends Do
static final byte BLOCK_COMPRESSED = 2;
static final byte GCD_COMPRESSED = 3;
- static final int VERSION_START = 3;
+ static final int VERSION_START = 4;
static final int VERSION_CURRENT = VERSION_START;
// clone for merge: when merging we don't do any instances.put()s
@@ -146,9 +146,8 @@ class MemoryDocValuesProducer extends Do
ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
boolean success = false;
try {
- version = CodecUtil.checkHeader(in, metaCodec,
- VERSION_START,
- VERSION_CURRENT);
+ version = CodecUtil.checkSegmentHeader(in, metaCodec, VERSION_START, VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
numEntries = readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in);
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
@@ -165,9 +164,8 @@ class MemoryDocValuesProducer extends Do
this.data = state.directory.openInput(dataName, state.context);
success = false;
try {
- final int version2 = CodecUtil.checkHeader(data, dataCodec,
- VERSION_START,
- VERSION_CURRENT);
+ final int version2 = CodecUtil.checkSegmentHeader(data, dataCodec, VERSION_START, VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
if (version != version2) {
throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
}
Modified: lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java Wed Oct 1 13:14:41 2014
@@ -99,15 +99,10 @@ public class IDVersionPostingsFormat ext
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new IDVersionPostingsReader();
+ PostingsReaderBase postingsReader = new IDVersionPostingsReader(state);
boolean success = false;
try {
- FieldsProducer ret = new VersionBlockTreeTermsReader(state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postingsReader,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new VersionBlockTreeTermsReader(postingsReader, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java Wed Oct 1 13:14:41 2014
@@ -26,21 +26,27 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.Bits;
final class IDVersionPostingsReader extends PostingsReaderBase {
+ final SegmentReadState state;
+
+ public IDVersionPostingsReader(SegmentReadState state) {
+ this.state = state;
+ }
@Override
public void init(IndexInput termsIn) throws IOException {
// Make sure we are talking to the matching postings writer
- CodecUtil.checkHeader(termsIn,
- IDVersionPostingsWriter.TERMS_CODEC,
- IDVersionPostingsWriter.VERSION_START,
- IDVersionPostingsWriter.VERSION_CURRENT);
+ CodecUtil.checkSegmentHeader(termsIn,
+ IDVersionPostingsWriter.TERMS_CODEC,
+ IDVersionPostingsWriter.VERSION_START,
+ IDVersionPostingsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
}
@Override
Modified: lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java Wed Oct 1 13:14:41 2014
@@ -26,7 +26,6 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
final class IDVersionPostingsWriter extends PushPostingsWriterBase {
@@ -34,7 +33,7 @@ final class IDVersionPostingsWriter exte
final static String TERMS_CODEC = "IDVersionPostingsWriterTerms";
// Increment version to change it
- final static int VERSION_START = 0;
+ final static int VERSION_START = 1;
final static int VERSION_CURRENT = VERSION_START;
final static IDVersionTermState emptyState = new IDVersionTermState();
@@ -57,7 +56,7 @@ final class IDVersionPostingsWriter exte
@Override
public void init(IndexOutput termsOut) throws IOException {
- CodecUtil.writeHeader(termsOut, TERMS_CODEC, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
}
@Override
Modified: lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java (original)
+++ lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java Wed Oct 1 13:14:41 2014
@@ -283,7 +283,7 @@ public final class IDVersionSegmentTerms
targetUpto = 0;
IDVersionSegmentTermsEnumFrame lastFrame = stack[0];
- assert validIndexPrefix <= term.length(): "validIndexPrefix=" + validIndexPrefix + " term.length=" + term.length() + " seg=" + fr.parent.segment;
+ assert validIndexPrefix <= term.length(): "validIndexPrefix=" + validIndexPrefix + " term.length=" + term.length() + " seg=" + fr.parent;
final int targetLimit = Math.min(target.length, validIndexPrefix);
@@ -1063,6 +1063,6 @@ public final class IDVersionSegmentTerms
@Override
public String toString() {
- return "IDVersionSegmentTermsEnum(seg=" + fr.parent.segment + ")";
+ return "IDVersionSegmentTermsEnum(seg=" + fr.parent + ")";
}
}
Modified: lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java Wed Oct 1 13:14:41 2014
@@ -29,12 +29,9 @@ import org.apache.lucene.codecs.FieldsPr
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
@@ -61,38 +58,36 @@ public final class VersionBlockTreeTerms
private final TreeMap<String,VersionFieldReader> fields = new TreeMap<>();
- /** File offset where the directory starts in the terms file. */
- private long dirOffset;
-
- /** File offset where the directory starts in the index file. */
- private long indexDirOffset;
-
- final String segment;
-
- private final int version;
-
/** Sole constructor. */
- public VersionBlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info,
- PostingsReaderBase postingsReader, IOContext ioContext,
- String segmentSuffix)
- throws IOException {
+ public VersionBlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
this.postingsReader = postingsReader;
- this.segment = info.name;
- in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VersionBlockTreeTermsWriter.TERMS_EXTENSION),
- ioContext);
+ String termsFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ VersionBlockTreeTermsWriter.TERMS_EXTENSION);
+ in = state.directory.openInput(termsFile, state.context);
boolean success = false;
IndexInput indexIn = null;
try {
- version = readHeader(in);
- indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION),
- ioContext);
- int indexVersion = readIndexHeader(indexIn);
- if (indexVersion != version) {
- throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion, indexIn);
+ int termsVersion = CodecUtil.checkSegmentHeader(in, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
+ VersionBlockTreeTermsWriter.VERSION_START,
+ VersionBlockTreeTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
+ String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION);
+ indexIn = state.directory.openInput(indexFile, state.context);
+ int indexVersion = CodecUtil.checkSegmentHeader(indexIn, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
+ VersionBlockTreeTermsWriter.VERSION_START,
+ VersionBlockTreeTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
+ if (indexVersion != termsVersion) {
+ throw new CorruptIndexException("mixmatched version files: " + in + "=" + termsVersion + "," + indexIn + "=" + indexVersion, indexIn);
}
// verify
@@ -108,8 +103,8 @@ public final class VersionBlockTreeTerms
CodecUtil.retrieveChecksum(in);
// Read per-field details
- seekDir(in, dirOffset);
- seekDir(indexIn, indexDirOffset);
+ seekDir(in);
+ seekDir(indexIn);
final int numFields = in.readVInt();
if (numFields < 0) {
@@ -126,7 +121,7 @@ public final class VersionBlockTreeTerms
code.length = numBytes;
final long version = in.readVLong();
final Pair<BytesRef,Long> rootCode = VersionBlockTreeTermsWriter.FST_OUTPUTS.newPair(code, version);
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
assert fieldInfo != null: "field=" + field;
final long sumTotalTermFreq = numTerms;
final long sumDocFreq = numTerms;
@@ -136,8 +131,8 @@ public final class VersionBlockTreeTerms
BytesRef minTerm = readBytesRef(in);
BytesRef maxTerm = readBytesRef(in);
- if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
- throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in);
+ if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
+ throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
}
if (sumDocFreq < docCount) { // #postings must be >= #docs with field
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
@@ -172,27 +167,10 @@ public final class VersionBlockTreeTerms
return bytes;
}
- /** Reads terms file header. */
- private int readHeader(IndexInput input) throws IOException {
- int version = CodecUtil.checkHeader(input, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
- VersionBlockTreeTermsWriter.VERSION_START,
- VersionBlockTreeTermsWriter.VERSION_CURRENT);
- return version;
- }
-
- /** Reads index file header. */
- private int readIndexHeader(IndexInput input) throws IOException {
- int version = CodecUtil.checkHeader(input, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
- VersionBlockTreeTermsWriter.VERSION_START,
- VersionBlockTreeTermsWriter.VERSION_CURRENT);
- return version;
- }
-
/** Seek {@code input} to the directory offset. */
- private void seekDir(IndexInput input, long dirOffset)
- throws IOException {
+ private void seekDir(IndexInput input) throws IOException {
input.seek(input.length() - CodecUtil.footerLength() - 8);
- dirOffset = input.readLong();
+ long dirOffset = input.readLong();
input.seek(dirOffset);
}
Modified: lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java?rev=1628697&r1=1628696&r2=1628697&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java Wed Oct 1 13:14:41 2014
@@ -121,7 +121,7 @@ public final class VersionBlockTreeTerms
final static String TERMS_CODEC_NAME = "VERSION_BLOCK_TREE_TERMS_DICT";
/** Initial terms format. */
- public static final int VERSION_START = 0;
+ public static final int VERSION_START = 1;
/** Current terms format. */
public static final int VERSION_CURRENT = VERSION_START;
@@ -199,13 +199,13 @@ public final class VersionBlockTreeTerms
fieldInfos = state.fieldInfos;
this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock;
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
//DEBUG = state.segmentName.equals("_4a");
final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexOut = state.directory.createOutput(termsIndexFileName, state.context);
- CodecUtil.writeHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter = postingsWriter;
// segment = state.segmentInfo.name;