You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/01 14:27:12 UTC
svn commit: r1628688 - in /lucene/dev/branches/lucene5969/lucene:
codecs/src/java/org/apache/lucene/codecs/blockterms/
test-framework/src/java/org/apache/lucene/codecs/lucene41ords/
test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/ test-...
Author: rmuir
Date: Wed Oct 1 12:27:11 2014
New Revision: 1628688
URL: http://svn.apache.org/r1628688
Log:
LUCENE-5969: clean up unnecessary back compat and add segment header
Modified:
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java
lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java
lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java Wed Oct 1 12:27:11 2014
@@ -33,15 +33,12 @@ import org.apache.lucene.index.DocsAndPo
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
@@ -108,24 +105,19 @@ public class BlockTermsReader extends Fi
}
}
- // private String segment;
-
- public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, SegmentInfo info, PostingsReaderBase postingsReader, IOContext context,
- String segmentSuffix)
- throws IOException {
+ public BlockTermsReader(TermsIndexReaderBase indexReader, PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
this.postingsReader = postingsReader;
-
- // this.segment = segment;
- in = dir.openInput(IndexFileNames.segmentFileName(info.name, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION),
- context);
+
+ String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, BlockTermsWriter.TERMS_EXTENSION);
+ in = state.directory.openInput(filename, state.context);
boolean success = false;
try {
CodecUtil.checkSegmentHeader(in, BlockTermsWriter.CODEC_NAME,
BlockTermsWriter.VERSION_START,
BlockTermsWriter.VERSION_CURRENT,
- info.getId(), segmentSuffix);
+ state.segmentInfo.getId(), state.segmentSuffix);
// Have PostingsReader init itself
postingsReader.init(in);
@@ -148,13 +140,13 @@ public class BlockTermsReader extends Fi
final long numTerms = in.readVLong();
assert numTerms >= 0;
final long termsStartPointer = in.readVLong();
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
final long sumDocFreq = in.readVLong();
final int docCount = in.readVInt();
final int longsSize = in.readVInt();
- if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
- throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in);
+ if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
+ throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
}
if (sumDocFreq < docCount) { // #postings must be >= #docs with field
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java Wed Oct 1 12:27:11 2014
@@ -59,7 +59,7 @@ import org.apache.lucene.util.RamUsageEs
public class BlockTermsWriter extends FieldsConsumer implements Closeable {
- final static String CODEC_NAME = "BLOCK_TERMS_DICT";
+ final static String CODEC_NAME = "BlockTermsWriter";
// Initial format
public static final int VERSION_START = 4;
@@ -281,9 +281,6 @@ public class BlockTermsWriter extends Fi
// EOF marker:
out.writeVInt(0);
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.sumDocFreq = sumDocFreq;
- this.docCount = docCount;
fieldIndexWriter.finish(out.getFilePointer());
if (numTerms > 0) {
fields.add(new FieldMetaData(fieldInfo,
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java Wed Oct 1 12:27:11 2014
@@ -24,9 +24,8 @@ import java.util.HashMap;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -46,27 +45,23 @@ public class VariableGapTermsIndexReader
final HashMap<String,FieldIndexData> fields = new HashMap<>();
- // start of the field info data
- private long dirOffset;
-
- private final int version;
-
- final String segment;
- public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, String segmentSuffix, IOContext context)
- throws IOException {
- final IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true));
- this.segment = segment;
+ public VariableGapTermsIndexReader(SegmentReadState state) throws IOException {
+ String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION);
+ final IndexInput in = state.directory.openInput(fileName, new IOContext(state.context, true));
boolean success = false;
try {
- version = readHeader(in);
+ CodecUtil.checkSegmentHeader(in, VariableGapTermsIndexWriter.CODEC_NAME,
+ VariableGapTermsIndexWriter.VERSION_START,
+ VariableGapTermsIndexWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
- if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(in);
- }
+ CodecUtil.checksumEntireFile(in);
- seekDir(in, dirOffset);
+ seekDir(in);
// Read directory
final int numFields = in.readVInt();
@@ -77,7 +72,7 @@ public class VariableGapTermsIndexReader
for(int i=0;i<numFields;i++) {
final int field = in.readVInt();
final long indexStart = in.readVLong();
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
FieldIndexData previous = fields.put(fieldInfo.name, new FieldIndexData(in, fieldInfo, indexStart));
if (previous != null) {
throw new CorruptIndexException("duplicate field: " + fieldInfo.name, in);
@@ -92,15 +87,6 @@ public class VariableGapTermsIndexReader
}
}
}
-
- private int readHeader(IndexInput input) throws IOException {
- int version = CodecUtil.checkHeader(input, VariableGapTermsIndexWriter.CODEC_NAME,
- VariableGapTermsIndexWriter.VERSION_START, VariableGapTermsIndexWriter.VERSION_CURRENT);
- if (version < VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) {
- dirOffset = input.readLong();
- }
- return version;
- }
private static class IndexEnum extends FieldIndexEnum {
private final BytesRefFSTEnum<Long> fstEnum;
@@ -206,14 +192,9 @@ public class VariableGapTermsIndexReader
@Override
public void close() throws IOException {}
- private void seekDir(IndexInput input, long dirOffset) throws IOException {
- if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) {
- input.seek(input.length() - CodecUtil.footerLength() - 8);
- dirOffset = input.readLong();
- } else if (version >= VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) {
- input.seek(input.length() - 8);
- dirOffset = input.readLong();
- }
+ private void seekDir(IndexInput input) throws IOException {
+ input.seek(input.length() - CodecUtil.footerLength() - 8);
+ long dirOffset = input.readLong();
input.seek(dirOffset);
}
Modified: lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java (original)
+++ lucene/dev/branches/lucene5969/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java Wed Oct 1 12:27:11 2014
@@ -52,11 +52,9 @@ public class VariableGapTermsIndexWriter
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tiv";
- final static String CODEC_NAME = "VARIABLE_GAP_TERMS_INDEX";
- final static int VERSION_START = 0;
- final static int VERSION_APPEND_ONLY = 1;
- final static int VERSION_CHECKSUM = 2;
- final static int VERSION_CURRENT = VERSION_CHECKSUM;
+ final static String CODEC_NAME = "VariableGapTermsIndex";
+ final static int VERSION_START = 3;
+ final static int VERSION_CURRENT = VERSION_START;
private final List<FSTFieldWriter> fields = new ArrayList<>();
@@ -184,7 +182,7 @@ public class VariableGapTermsIndexWriter
try {
fieldInfos = state.fieldInfos;
this.policy = policy;
- writeHeader(out);
+ CodecUtil.writeSegmentHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true;
} finally {
if (!success) {
@@ -192,10 +190,6 @@ public class VariableGapTermsIndexWriter
}
}
}
-
- private void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- }
@Override
public FieldWriter addField(FieldInfo field, long termsFilePointer) throws IOException {
Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41ords/Lucene41WithOrds.java Wed Oct 1 12:27:11 2014
@@ -35,7 +35,6 @@ import org.apache.lucene.codecs.lucene41
import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.BytesRef;
// TODO: we could make separate base class that can wrapp
// any PostingsBaseFormat and make it ord-able...
@@ -110,13 +109,7 @@ public final class Lucene41WithOrds exte
success = false;
try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postings,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapDocFreqInterval.java Wed Oct 1 12:27:11 2014
@@ -103,10 +103,7 @@ public final class Lucene41VarGapDocFreq
boolean success = false;
try {
- indexReader = new VariableGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- state.segmentSuffix, state.context);
+ indexReader = new VariableGapTermsIndexReader(state);
success = true;
} finally {
if (!success) {
@@ -116,13 +113,7 @@ public final class Lucene41VarGapDocFreq
success = false;
try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postings,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene41vargap/Lucene41VarGapFixedInterval.java Wed Oct 1 12:27:11 2014
@@ -100,10 +100,7 @@ public final class Lucene41VarGapFixedIn
boolean success = false;
try {
- indexReader = new VariableGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- state.segmentSuffix, state.context);
+ indexReader = new VariableGapTermsIndexReader(state);
success = true;
} finally {
if (!success) {
@@ -113,13 +110,7 @@ public final class Lucene41VarGapFixedIn
success = false;
try {
- FieldsProducer ret = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postings,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new BlockTermsReader(indexReader, postings, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1628688&r1=1628687&r2=1628688&view=diff
==============================================================================
--- lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/lucene5969/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Wed Oct 1 12:27:11 2014
@@ -351,10 +351,7 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: variable-gap terms index");
}
- indexReader = new VariableGapTermsIndexReader(state.directory,
- state.fieldInfos,
- state.segmentInfo.name,
- state.segmentSuffix, state.context);
+ indexReader = new VariableGapTermsIndexReader(state);
}
@@ -367,13 +364,7 @@ public final class MockRandomPostingsFor
success = false;
try {
- fields = new BlockTermsReader(indexReader,
- state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postingsReader,
- state.context,
- state.segmentSuffix);
+ fields = new BlockTermsReader(indexReader, postingsReader, state);
success = true;
} finally {
if (!success) {