You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/05 16:43:51 UTC
svn commit: r1629501 [4/7] - in /lucene/dev/branches/branch_5x: ./
dev-tools/ lucene/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/
lucene/backward-codecs/src/java/...
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWDocValuesFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWDocValuesFormat.java Sun Oct 5 14:43:47 2014
@@ -21,13 +21,13 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.LuceneTestCase;
/**
- * Read-write version of {@link Lucene42DocValuesFormat} for testing.
+ * Read-Write version of 4.2 docvalues format for testing
+ * @deprecated for test purposes only
*/
-@SuppressWarnings("deprecation")
-public class Lucene42RWDocValuesFormat extends Lucene42DocValuesFormat {
+@Deprecated
+public final class Lucene42RWDocValuesFormat extends Lucene42DocValuesFormat {
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWNormsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWNormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWNormsFormat.java Sun Oct 5 14:43:47 2014
@@ -21,12 +21,13 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.LuceneTestCase;
/**
- * Read-write version of {@link Lucene42NormsFormat}
+ * Read-write version of 4.2 norms format for testing
+ * @deprecated for test purposes only
*/
-public class Lucene42RWNormsFormat extends Lucene42NormsFormat {
+@Deprecated
+public final class Lucene42RWNormsFormat extends Lucene42NormsFormat {
@Override
public NormsConsumer normsConsumer(SegmentWriteState state) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWCodec.java Sun Oct 5 14:43:47 2014
@@ -24,16 +24,20 @@ import org.apache.lucene.codecs.FieldInf
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40RWSegmentInfoFormat;
+import org.apache.lucene.codecs.lucene41.Lucene41RWStoredFieldsFormat;
import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosFormat;
import org.apache.lucene.codecs.lucene42.Lucene42FieldInfosWriter;
import org.apache.lucene.codecs.lucene42.Lucene42RWNormsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42RWTermVectorsFormat;
/**
* Read-write version of {@link Lucene45Codec} for testing.
*/
@SuppressWarnings("deprecation")
-public class Lucene45RWCodec extends Lucene45Codec {
+public final class Lucene45RWCodec extends Lucene45Codec {
private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat() {
@Override
@@ -67,4 +71,18 @@ public class Lucene45RWCodec extends Luc
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfos;
}
+
+ private static final StoredFieldsFormat storedFields = new Lucene41RWStoredFieldsFormat();
+
+ @Override
+ public StoredFieldsFormat storedFieldsFormat() {
+ return storedFields;
+ }
+
+ private final TermVectorsFormat vectorsFormat = new Lucene42RWTermVectorsFormat();
+
+ @Override
+ public TermVectorsFormat termVectorsFormat() {
+ return vectorsFormat;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWDocValuesFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/Lucene45RWDocValuesFormat.java Sun Oct 5 14:43:47 2014
@@ -24,9 +24,11 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.SegmentWriteState;
/**
- * Read-write version of {@link Lucene45DocValuesFormat} for testing.
+ * Read-write version of 4.5 docvalues format for testing
+ * @deprecated for test purposes only
*/
-public class Lucene45RWDocValuesFormat extends Lucene45DocValuesFormat {
+@Deprecated
+public final class Lucene45RWDocValuesFormat extends Lucene45DocValuesFormat {
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene45/TestLucene45DocValuesFormat.java Sun Oct 5 14:43:47 2014
@@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase;
-import org.junit.BeforeClass;
/**
* Tests Lucene45DocValuesFormat
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWCodec.java Sun Oct 5 14:43:47 2014
@@ -20,15 +20,19 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
-import org.apache.lucene.codecs.SegmentInfoWriter;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene41.Lucene41RWStoredFieldsFormat;
import org.apache.lucene.codecs.lucene42.Lucene42RWNormsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42RWTermVectorsFormat;
import org.apache.lucene.codecs.lucene45.Lucene45RWDocValuesFormat;
/**
- * Read-write version of {@link Lucene46Codec} for testing.
+ * Read-write version of 4.6 codec for testing
+ * @deprecated for test purposes only
*/
-@SuppressWarnings("deprecation")
-public class Lucene46RWCodec extends Lucene46Codec {
+@Deprecated
+public final class Lucene46RWCodec extends Lucene46Codec {
private static final DocValuesFormat docValues = new Lucene45RWDocValuesFormat();
@@ -50,4 +54,18 @@ public class Lucene46RWCodec extends Luc
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfos;
}
+
+ private static final StoredFieldsFormat storedFields = new Lucene41RWStoredFieldsFormat();
+
+ @Override
+ public StoredFieldsFormat storedFieldsFormat() {
+ return storedFields;
+ }
+
+ private final TermVectorsFormat vectorsFormat = new Lucene42RWTermVectorsFormat();
+
+ @Override
+ public TermVectorsFormat termVectorsFormat() {
+ return vectorsFormat;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWSegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWSegmentInfoFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWSegmentInfoFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46RWSegmentInfoFormat.java Sun Oct 5 14:43:47 2014
@@ -19,8 +19,12 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.SegmentInfoWriter;
-/** read-write version of 4.6 segmentinfos for testing */
-public class Lucene46RWSegmentInfoFormat extends Lucene46SegmentInfoFormat {
+/**
+ * Read-Write version of 4.6 segmentinfo format for testing
+ * @deprecated for test purposes only
+ */
+@Deprecated
+public final class Lucene46RWSegmentInfoFormat extends Lucene46SegmentInfoFormat {
@Override
public SegmentInfoWriter getSegmentInfoWriter() {
return new Lucene46SegmentInfoWriter();
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/Lucene46SegmentInfoWriter.java Sun Oct 5 14:43:47 2014
@@ -31,12 +31,11 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
- * Lucene 4.0 implementation of {@link SegmentInfoWriter}.
- *
- * @see Lucene46SegmentInfoFormat
- * @lucene.experimental
+ * Writer for 4.0 segmentinfo format for testing
+ * @deprecated for test purposes only
*/
-public class Lucene46SegmentInfoWriter extends SegmentInfoWriter {
+@Deprecated
+final class Lucene46SegmentInfoWriter extends SegmentInfoWriter {
/** Sole constructor. */
public Lucene46SegmentInfoWriter() {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWCodec.java Sun Oct 5 14:43:47 2014
@@ -20,13 +20,18 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene41.Lucene41RWStoredFieldsFormat;
+import org.apache.lucene.codecs.lucene42.Lucene42RWTermVectorsFormat;
import org.apache.lucene.codecs.lucene46.Lucene46RWSegmentInfoFormat;
/**
- * Read-write version of {@link Lucene49Codec} for testing.
+ * Read-Write version of 4.9 codec for testing
+ * @deprecated for test purposes only
*/
-@SuppressWarnings("deprecation")
-public class Lucene49RWCodec extends Lucene49Codec {
+@Deprecated
+public final class Lucene49RWCodec extends Lucene49Codec {
private static final DocValuesFormat docValues = new Lucene49RWDocValuesFormat();
@@ -35,7 +40,7 @@ public class Lucene49RWCodec extends Luc
return docValues;
}
- private static final NormsFormat norms = new Lucene49NormsFormat();
+ private static final NormsFormat norms = new Lucene49RWNormsFormat();
@Override
public NormsFormat normsFormat() {
@@ -48,4 +53,18 @@ public class Lucene49RWCodec extends Luc
public SegmentInfoFormat segmentInfoFormat() {
return segmentInfos;
}
+
+ private static final StoredFieldsFormat storedFields = new Lucene41RWStoredFieldsFormat();
+
+ @Override
+ public StoredFieldsFormat storedFieldsFormat() {
+ return storedFields;
+ }
+
+ private final TermVectorsFormat vectorsFormat = new Lucene42RWTermVectorsFormat();
+
+ @Override
+ public TermVectorsFormat termVectorsFormat() {
+ return vectorsFormat;
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWDocValuesFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWDocValuesFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/Lucene49RWDocValuesFormat.java Sun Oct 5 14:43:47 2014
@@ -22,10 +22,13 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.LuceneTestCase;
-/** Read-write version of {@link Lucene49DocValuesFormat} for testing */
-public class Lucene49RWDocValuesFormat extends Lucene49DocValuesFormat {
+/**
+ * Read-Write version of 4.9 docvalues format for testing
+ * @deprecated for test purposes only
+ */
+@Deprecated
+public final class Lucene49RWDocValuesFormat extends Lucene49DocValuesFormat {
@Override
public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49DocValuesFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49DocValuesFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49DocValuesFormat.java Sun Oct 5 14:43:47 2014
@@ -19,7 +19,6 @@ package org.apache.lucene.codecs.lucene4
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase;
-import org.junit.BeforeClass;
/**
* Tests Lucene49DocValuesFormat
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java Sun Oct 5 14:43:47 2014
@@ -33,15 +33,12 @@ import org.apache.lucene.index.DocsAndPo
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
@@ -77,12 +74,7 @@ public class BlockTermsReader extends Fi
// Reads the terms index
private TermsIndexReaderBase indexReader;
-
- // keeps the dirStart offset
- private long dirOffset;
- private final int version;
-
// Used as key for the terms cache
private static class FieldAndTerm extends DoubleBarrelLRUCache.CloneableKey {
String field;
@@ -113,35 +105,31 @@ public class BlockTermsReader extends Fi
}
}
- // private String segment;
-
- public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, SegmentInfo info, PostingsReaderBase postingsReader, IOContext context,
- String segmentSuffix)
- throws IOException {
+ public BlockTermsReader(TermsIndexReaderBase indexReader, PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
this.postingsReader = postingsReader;
-
- // this.segment = segment;
- in = dir.openInput(IndexFileNames.segmentFileName(info.name, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION),
- context);
+
+ String filename = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, BlockTermsWriter.TERMS_EXTENSION);
+ in = state.directory.openInput(filename, state.context);
boolean success = false;
try {
- version = readHeader(in);
+ CodecUtil.checkSegmentHeader(in, BlockTermsWriter.CODEC_NAME,
+ BlockTermsWriter.VERSION_START,
+ BlockTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
// Have PostingsReader init itself
postingsReader.init(in);
- if (version >= BlockTermsWriter.VERSION_CHECKSUM) {
- // NOTE: data file is too costly to verify checksum against all the bytes on open,
- // but for now we at least verify proper structure of the checksum footer: which looks
- // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
- // such as file truncation.
- CodecUtil.retrieveChecksum(in);
- }
+ // NOTE: data file is too costly to verify checksum against all the bytes on open,
+ // but for now we at least verify proper structure of the checksum footer: which looks
+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+ // such as file truncation.
+ CodecUtil.retrieveChecksum(in);
// Read per-field details
- seekDir(in, dirOffset);
+ seekDir(in);
final int numFields = in.readVInt();
if (numFields < 0) {
@@ -152,13 +140,13 @@ public class BlockTermsReader extends Fi
final long numTerms = in.readVLong();
assert numTerms >= 0;
final long termsStartPointer = in.readVLong();
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
final long sumDocFreq = in.readVLong();
final int docCount = in.readVInt();
- final int longsSize = version >= BlockTermsWriter.VERSION_META_ARRAY ? in.readVInt() : 0;
- if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
- throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in);
+ final int longsSize = in.readVInt();
+ if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
+ throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
}
if (sumDocFreq < docCount) { // #postings must be >= #docs with field
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
@@ -180,25 +168,10 @@ public class BlockTermsReader extends Fi
this.indexReader = indexReader;
}
-
- private int readHeader(IndexInput input) throws IOException {
- int version = CodecUtil.checkHeader(input, BlockTermsWriter.CODEC_NAME,
- BlockTermsWriter.VERSION_START,
- BlockTermsWriter.VERSION_CURRENT);
- if (version < BlockTermsWriter.VERSION_APPEND_ONLY) {
- dirOffset = input.readLong();
- }
- return version;
- }
- private void seekDir(IndexInput input, long dirOffset) throws IOException {
- if (version >= BlockTermsWriter.VERSION_CHECKSUM) {
- input.seek(input.length() - CodecUtil.footerLength() - 8);
- dirOffset = input.readLong();
- } else if (version >= BlockTermsWriter.VERSION_APPEND_ONLY) {
- input.seek(input.length() - 8);
- dirOffset = input.readLong();
- }
+ private void seekDir(IndexInput input) throws IOException {
+ input.seek(input.length() - CodecUtil.footerLength() - 8);
+ long dirOffset = input.readLong();
input.seek(dirOffset);
}
@@ -906,9 +879,8 @@ public class BlockTermsReader extends Fi
@Override
public void checkIntegrity() throws IOException {
// verify terms
- if (version >= BlockTermsWriter.VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(in);
- }
+ CodecUtil.checksumEntireFile(in);
+
// verify postings
postingsReader.checkIntegrity();
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java Sun Oct 5 14:43:47 2014
@@ -59,14 +59,11 @@ import org.apache.lucene.util.RamUsageEs
public class BlockTermsWriter extends FieldsConsumer implements Closeable {
- final static String CODEC_NAME = "BLOCK_TERMS_DICT";
+ final static String CODEC_NAME = "BlockTermsWriter";
// Initial format
- public static final int VERSION_START = 0;
- public static final int VERSION_APPEND_ONLY = 1;
- public static final int VERSION_META_ARRAY = 2;
- public static final int VERSION_CHECKSUM = 3;
- public static final int VERSION_CURRENT = VERSION_CHECKSUM;
+ public static final int VERSION_START = 4;
+ public static final int VERSION_CURRENT = VERSION_START;
/** Extension of terms file */
static final String TERMS_EXTENSION = "tib";
@@ -113,7 +110,7 @@ public class BlockTermsWriter extends Fi
boolean success = false;
try {
fieldInfos = state.fieldInfos;
- writeHeader(out);
+ CodecUtil.writeSegmentHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
currentField = null;
this.postingsWriter = postingsWriter;
// segment = state.segmentName;
@@ -128,10 +125,6 @@ public class BlockTermsWriter extends Fi
}
}
}
-
- private void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- }
@Override
public void write(Fields fields) throws IOException {
@@ -184,9 +177,7 @@ public class BlockTermsWriter extends Fi
}
out.writeVLong(field.sumDocFreq);
out.writeVInt(field.docCount);
- if (VERSION_CURRENT >= VERSION_META_ARRAY) {
- out.writeVInt(field.longsSize);
- }
+ out.writeVInt(field.longsSize);
}
writeTrailer(dirStart);
CodecUtil.writeFooter(out);
@@ -290,9 +281,6 @@ public class BlockTermsWriter extends Fi
// EOF marker:
out.writeVInt(0);
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.sumDocFreq = sumDocFreq;
- this.docCount = docCount;
fieldIndexWriter.finish(out.getFilePointer());
if (numTerms > 0) {
fields.add(new FieldMetaData(fieldInfo,
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexReader.java Sun Oct 5 14:43:47 2014
@@ -17,13 +17,11 @@ package org.apache.lucene.codecs.blockte
* limitations under the License.
*/
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
import org.apache.lucene.util.BytesRef;
@@ -34,7 +32,6 @@ import org.apache.lucene.util.packed.Mon
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Comparator;
import java.util.List;
import java.io.IOException;
@@ -58,8 +55,6 @@ public class FixedGapTermsIndexReader ex
private final int packedIntsVersion;
private final int blocksize;
- private final Comparator<BytesRef> termComp;
-
private final static int PAGED_BYTES_BITS = 15;
// all fields share this single logical byte[]
@@ -67,28 +62,24 @@ public class FixedGapTermsIndexReader ex
final HashMap<String,FieldIndexData> fields = new HashMap<>();
- // start of the field info data
- private long dirOffset;
-
- private int version;
-
- public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, Comparator<BytesRef> termComp, String segmentSuffix, IOContext context)
- throws IOException {
+ public FixedGapTermsIndexReader(SegmentReadState state) throws IOException {
final PagedBytes termBytes = new PagedBytes(PAGED_BYTES_BITS);
-
- this.termComp = termComp;
- final IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context);
+ String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION);
+ final IndexInput in = state.directory.openInput(fileName, state.context);
boolean success = false;
try {
- readHeader(in);
+ CodecUtil.checkSegmentHeader(in, FixedGapTermsIndexWriter.CODEC_NAME,
+ FixedGapTermsIndexWriter.VERSION_CURRENT,
+ FixedGapTermsIndexWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
- if (version >= FixedGapTermsIndexWriter.VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(in);
- }
+ CodecUtil.checksumEntireFile(in);
indexInterval = in.readVInt();
if (indexInterval < 1) {
@@ -97,7 +88,7 @@ public class FixedGapTermsIndexReader ex
packedIntsVersion = in.readVInt();
blocksize = in.readVInt();
- seekDir(in, dirOffset);
+ seekDir(in);
// Read directory
final int numFields = in.readVInt();
@@ -118,7 +109,7 @@ public class FixedGapTermsIndexReader ex
if (packedIndexStart < indexStart) {
throw new CorruptIndexException("invalid packedIndexStart: " + packedIndexStart + " indexStart: " + indexStart + "numIndexTerms: " + numIndexTerms, in);
}
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
FieldIndexData previous = fields.put(fieldInfo.name, new FieldIndexData(in, termBytes, indexStart, termsStart, packedIndexStart, packedOffsetsStart, numIndexTerms));
if (previous != null) {
throw new CorruptIndexException("duplicate field: " + fieldInfo.name, in);
@@ -135,11 +126,6 @@ public class FixedGapTermsIndexReader ex
}
}
- private void readHeader(IndexInput input) throws IOException {
- version = CodecUtil.checkHeader(input, FixedGapTermsIndexWriter.CODEC_NAME,
- FixedGapTermsIndexWriter.VERSION_CURRENT, FixedGapTermsIndexWriter.VERSION_CURRENT);
- }
-
private class IndexEnum extends FieldIndexEnum {
private final FieldIndexData fieldIndex;
private final BytesRef term = new BytesRef();
@@ -166,7 +152,7 @@ public class FixedGapTermsIndexReader ex
final int length = (int) (fieldIndex.termOffsets.get(1+mid) - offset);
termBytesReader.fillSlice(term, fieldIndex.termBytesStart + offset, length);
- int delta = termComp.compare(target, term);
+ int delta = target.compareTo(term);
if (delta < 0) {
hi = mid - 1;
} else if (delta > 0) {
@@ -301,13 +287,9 @@ public class FixedGapTermsIndexReader ex
@Override
public void close() throws IOException {}
- private void seekDir(IndexInput input, long dirOffset) throws IOException {
- if (version >= FixedGapTermsIndexWriter.VERSION_CHECKSUM) {
- input.seek(input.length() - CodecUtil.footerLength() - 8);
- } else {
- input.seek(input.length() - 8);
- }
- dirOffset = input.readLong();
+ private void seekDir(IndexInput input) throws IOException {
+ input.seek(input.length() - CodecUtil.footerLength() - 8);
+ long dirOffset = input.readLong();
input.seek(dirOffset);
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/FixedGapTermsIndexWriter.java Sun Oct 5 14:43:47 2014
@@ -49,12 +49,9 @@ public class FixedGapTermsIndexWriter ex
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tii";
- final static String CODEC_NAME = "SIMPLE_STANDARD_TERMS_INDEX";
- final static int VERSION_START = 0;
- final static int VERSION_APPEND_ONLY = 1;
- final static int VERSION_MONOTONIC_ADDRESSING = 2;
- final static int VERSION_CHECKSUM = 3;
- final static int VERSION_CURRENT = VERSION_CHECKSUM;
+ final static String CODEC_NAME = "FixedGapTermsIndex";
+ final static int VERSION_START = 4;
+ final static int VERSION_CURRENT = VERSION_START;
final static int BLOCKSIZE = 4096;
final private int termIndexInterval;
@@ -75,7 +72,7 @@ public class FixedGapTermsIndexWriter ex
out = state.directory.createOutput(indexFileName, state.context);
boolean success = false;
try {
- writeHeader(out);
+ CodecUtil.writeSegmentHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
out.writeVInt(termIndexInterval);
out.writeVInt(PackedInts.VERSION_CURRENT);
out.writeVInt(BLOCKSIZE);
@@ -86,10 +83,6 @@ public class FixedGapTermsIndexWriter ex
}
}
}
-
- private void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- }
@Override
public FieldWriter addField(FieldInfo field, long termsFilePointer) {
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexReader.java Sun Oct 5 14:43:47 2014
@@ -24,9 +24,8 @@ import java.util.HashMap;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.store.Directory;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
@@ -46,27 +45,23 @@ public class VariableGapTermsIndexReader
final HashMap<String,FieldIndexData> fields = new HashMap<>();
- // start of the field info data
- private long dirOffset;
-
- private final int version;
-
- final String segment;
- public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, String segmentSuffix, IOContext context)
- throws IOException {
- final IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true));
- this.segment = segment;
+ public VariableGapTermsIndexReader(SegmentReadState state) throws IOException {
+ String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION);
+ final IndexInput in = state.directory.openInput(fileName, new IOContext(state.context, true));
boolean success = false;
try {
- version = readHeader(in);
+ CodecUtil.checkSegmentHeader(in, VariableGapTermsIndexWriter.CODEC_NAME,
+ VariableGapTermsIndexWriter.VERSION_START,
+ VariableGapTermsIndexWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
- if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(in);
- }
+ CodecUtil.checksumEntireFile(in);
- seekDir(in, dirOffset);
+ seekDir(in);
// Read directory
final int numFields = in.readVInt();
@@ -77,7 +72,7 @@ public class VariableGapTermsIndexReader
for(int i=0;i<numFields;i++) {
final int field = in.readVInt();
final long indexStart = in.readVLong();
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
FieldIndexData previous = fields.put(fieldInfo.name, new FieldIndexData(in, fieldInfo, indexStart));
if (previous != null) {
throw new CorruptIndexException("duplicate field: " + fieldInfo.name, in);
@@ -92,15 +87,6 @@ public class VariableGapTermsIndexReader
}
}
}
-
- private int readHeader(IndexInput input) throws IOException {
- int version = CodecUtil.checkHeader(input, VariableGapTermsIndexWriter.CODEC_NAME,
- VariableGapTermsIndexWriter.VERSION_START, VariableGapTermsIndexWriter.VERSION_CURRENT);
- if (version < VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) {
- dirOffset = input.readLong();
- }
- return version;
- }
private static class IndexEnum extends FieldIndexEnum {
private final BytesRefFSTEnum<Long> fstEnum;
@@ -206,14 +192,9 @@ public class VariableGapTermsIndexReader
@Override
public void close() throws IOException {}
- private void seekDir(IndexInput input, long dirOffset) throws IOException {
- if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) {
- input.seek(input.length() - CodecUtil.footerLength() - 8);
- dirOffset = input.readLong();
- } else if (version >= VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) {
- input.seek(input.length() - 8);
- dirOffset = input.readLong();
- }
+ private void seekDir(IndexInput input) throws IOException {
+ input.seek(input.length() - CodecUtil.footerLength() - 8);
+ long dirOffset = input.readLong();
input.seek(dirOffset);
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java Sun Oct 5 14:43:47 2014
@@ -52,11 +52,9 @@ public class VariableGapTermsIndexWriter
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tiv";
- final static String CODEC_NAME = "VARIABLE_GAP_TERMS_INDEX";
- final static int VERSION_START = 0;
- final static int VERSION_APPEND_ONLY = 1;
- final static int VERSION_CHECKSUM = 2;
- final static int VERSION_CURRENT = VERSION_CHECKSUM;
+ final static String CODEC_NAME = "VariableGapTermsIndex";
+ final static int VERSION_START = 3;
+ final static int VERSION_CURRENT = VERSION_START;
private final List<FSTFieldWriter> fields = new ArrayList<>();
@@ -184,7 +182,7 @@ public class VariableGapTermsIndexWriter
try {
fieldInfos = state.fieldInfos;
this.policy = policy;
- writeHeader(out);
+ CodecUtil.writeSegmentHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true;
} finally {
if (!success) {
@@ -192,10 +190,6 @@ public class VariableGapTermsIndexWriter
}
}
}
-
- private void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- }
@Override
public FieldWriter addField(FieldInfo field, long termsFilePointer) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/Ords41PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/Ords41PostingsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/Ords41PostingsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/Ords41PostingsFormat.java Sun Oct 5 14:43:47 2014
@@ -94,12 +94,7 @@ public class Ords41PostingsFormat extend
state.segmentSuffix);
boolean success = false;
try {
- FieldsProducer ret = new OrdsBlockTreeTermsReader(state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postingsReader,
- state.context,
- state.segmentSuffix);
+ FieldsProducer ret = new OrdsBlockTreeTermsReader(postingsReader, state);
success = true;
return ret;
} finally {
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsReader.java Sun Oct 5 14:43:47 2014
@@ -31,12 +31,9 @@ import org.apache.lucene.codecs.blocktre
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Accountables;
@@ -62,42 +59,33 @@ public final class OrdsBlockTreeTermsRea
private final TreeMap<String,OrdsFieldReader> fields = new TreeMap<>();
- /** File offset where the directory starts in the terms file. */
- private long dirOffset;
-
- /** File offset where the directory starts in the index file. */
- private long indexDirOffset;
-
- final String segment;
-
- private final int version;
-
/** Sole constructor. */
- public OrdsBlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo info,
- PostingsReaderBase postingsReader, IOContext ioContext,
- String segmentSuffix)
- throws IOException {
+ public OrdsBlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state) throws IOException {
this.postingsReader = postingsReader;
- this.segment = info.name;
- in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, OrdsBlockTreeTermsWriter.TERMS_EXTENSION),
- ioContext);
+ String termsFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ OrdsBlockTreeTermsWriter.TERMS_EXTENSION);
+ in = state.directory.openInput(termsFile, state.context);
boolean success = false;
IndexInput indexIn = null;
try {
- version = CodecUtil.checkHeader(in,
- OrdsBlockTreeTermsWriter.TERMS_CODEC_NAME,
- OrdsBlockTreeTermsWriter.VERSION_START,
- OrdsBlockTreeTermsWriter.VERSION_CURRENT);
- indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, OrdsBlockTreeTermsWriter.TERMS_INDEX_EXTENSION),
- ioContext);
- int indexVersion = CodecUtil.checkHeader(indexIn,
- OrdsBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
- OrdsBlockTreeTermsWriter.VERSION_START,
- OrdsBlockTreeTermsWriter.VERSION_CURRENT);
+ int version = CodecUtil.checkSegmentHeader(in, OrdsBlockTreeTermsWriter.TERMS_CODEC_NAME,
+ OrdsBlockTreeTermsWriter.VERSION_START,
+ OrdsBlockTreeTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
+ String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name,
+ state.segmentSuffix,
+ OrdsBlockTreeTermsWriter.TERMS_INDEX_EXTENSION);
+ indexIn = state.directory.openInput(indexFile, state.context);
+ int indexVersion = CodecUtil.checkSegmentHeader(indexIn, OrdsBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
+ OrdsBlockTreeTermsWriter.VERSION_START,
+ OrdsBlockTreeTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
if (indexVersion != version) {
throw new CorruptIndexException("mixmatched version files: " + in + "=" + version + "," + indexIn + "=" + indexVersion, indexIn);
}
@@ -116,8 +104,8 @@ public final class OrdsBlockTreeTermsRea
CodecUtil.retrieveChecksum(in);
// Read per-field details
- seekDir(in, dirOffset);
- seekDir(indexIn, indexDirOffset);
+ seekDir(in);
+ seekDir(indexIn);
final int numFields = in.readVInt();
if (numFields < 0) {
@@ -134,7 +122,7 @@ public final class OrdsBlockTreeTermsRea
in.readBytes(code.bytes, 0, numBytes);
code.length = numBytes;
final Output rootCode = OrdsBlockTreeTermsWriter.FST_OUTPUTS.newOutput(code, 0, numTerms);
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
assert fieldInfo != null: "field=" + field;
assert numTerms <= Integer.MAX_VALUE;
final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
@@ -145,8 +133,8 @@ public final class OrdsBlockTreeTermsRea
BytesRef minTerm = readBytesRef(in);
BytesRef maxTerm = readBytesRef(in);
- if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
- throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount(), in);
+ if (docCount < 0 || docCount > state.segmentInfo.getDocCount()) { // #docs with field must be <= #docs
+ throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + state.segmentInfo.getDocCount(), in);
}
if (sumDocFreq < docCount) { // #postings must be >= #docs with field
throw new CorruptIndexException("invalid sumDocFreq: " + sumDocFreq + " docCount: " + docCount, in);
@@ -182,10 +170,9 @@ public final class OrdsBlockTreeTermsRea
}
/** Seek {@code input} to the directory offset. */
- private void seekDir(IndexInput input, long dirOffset)
- throws IOException {
+ private void seekDir(IndexInput input) throws IOException {
input.seek(input.length() - CodecUtil.footerLength() - 8);
- dirOffset = input.readLong();
+ long dirOffset = input.readLong();
input.seek(dirOffset);
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java Sun Oct 5 14:43:47 2014
@@ -42,7 +42,6 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.fst.Builder;
@@ -115,17 +114,17 @@ public final class OrdsBlockTreeTermsWri
/** Extension of terms file */
static final String TERMS_EXTENSION = "tio";
- final static String TERMS_CODEC_NAME = "BLOCK_TREE_ORDS_TERMS_DICT";
+ final static String TERMS_CODEC_NAME = "OrdsBlockTreeTerms";
/** Initial terms format. */
- public static final int VERSION_START = 0;
+ public static final int VERSION_START = 1;
/** Current terms format. */
public static final int VERSION_CURRENT = VERSION_START;
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tipo";
- final static String TERMS_INDEX_CODEC_NAME = "BLOCK_TREE_ORDS_TERMS_INDEX";
+ final static String TERMS_INDEX_CODEC_NAME = "OrdsBlockTreeIndex";
private final IndexOutput out;
private final IndexOutput indexOut;
@@ -204,11 +203,11 @@ public final class OrdsBlockTreeTermsWri
fieldInfos = state.fieldInfos;
this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock;
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexOut = state.directory.createOutput(termsIndexFileName, state.context);
- CodecUtil.writeHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter = postingsWriter;
// segment = state.segmentInfo.name;
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java Sun Oct 5 14:43:47 2014
@@ -1030,7 +1030,7 @@ public final class OrdsSegmentTermsEnum
@Override
public String toString() {
- return "OrdsSegmentTermsEnum(seg=" + fr.parent.segment + ")";
+ return "OrdsSegmentTermsEnum(seg=" + fr.parent + ")";
}
/** Holds a single input (IntsRef) + output pair. */
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Sun Oct 5 14:43:47 2014
@@ -72,7 +72,7 @@ import org.apache.lucene.util.automaton.
* NumFilteredFields, Filter<sup>NumFilteredFields</sup>, Footer</li>
* <li>Filter --> FieldNumber, FuzzySet</li>
* <li>FuzzySet -->See {@link FuzzySet#serialize(DataOutput)}</li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
* <li>DelegatePostingsFormatName --> {@link DataOutput#writeString(String)
* String} The name of a ServiceProvider registered {@link PostingsFormat}</li>
* <li>NumFilteredFields --> {@link DataOutput#writeInt Uint32}</li>
@@ -85,9 +85,8 @@ import org.apache.lucene.util.automaton.
public final class BloomFilteringPostingsFormat extends PostingsFormat {
public static final String BLOOM_CODEC_NAME = "BloomFilter";
- public static final int VERSION_START = 1;
- public static final int VERSION_CHECKSUM = 2;
- public static final int VERSION_CURRENT = VERSION_CHECKSUM;
+ public static final int VERSION_START = 3;
+ public static final int VERSION_CURRENT = VERSION_START;
/** Extension of Bloom Filters file */
static final String BLOOM_EXTENSION = "blm";
@@ -167,7 +166,7 @@ public final class BloomFilteringPosting
boolean success = false;
try {
bloomIn = state.directory.openChecksumInput(bloomFileName, state.context);
- int version = CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT);
+ CodecUtil.checkSegmentHeader(bloomIn, BLOOM_CODEC_NAME, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
// // Load the hash function used in the BloomFilter
// hashFunction = HashFunction.forName(bloomIn.readString());
// Load the delegate postings format
@@ -183,11 +182,7 @@ public final class BloomFilteringPosting
FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNum);
bloomsByFieldName.put(fieldInfo.name, bloom);
}
- if (version >= VERSION_CHECKSUM) {
- CodecUtil.checkFooter(bloomIn);
- } else {
- CodecUtil.checkEOF(bloomIn);
- }
+ CodecUtil.checkFooter(bloomIn);
IOUtils.close(bloomIn);
success = true;
} finally {
@@ -507,7 +502,7 @@ public final class BloomFilteringPosting
IndexOutput bloomOutput = null;
try {
bloomOutput = state.directory.createOutput(bloomFileName, state.context);
- CodecUtil.writeHeader(bloomOutput, BLOOM_CODEC_NAME, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(bloomOutput, BLOOM_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
// remember the name of the postings format we will delegate to
bloomOutput.writeString(delegatePostingsFormat.getName());
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesConsumer.java Sun Oct 5 14:43:47 2014
@@ -52,10 +52,10 @@ class DirectDocValuesConsumer extends Do
try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java Sun Oct 5 14:43:47 2014
@@ -76,6 +76,8 @@ class DirectDocValuesProducer extends Do
private final AtomicLong ramBytesUsed;
private final int version;
+ private final boolean merging;
+
static final byte NUMBER = 0;
static final byte BYTES = 1;
static final byte SORTED = 2;
@@ -84,20 +86,44 @@ class DirectDocValuesProducer extends Do
static final byte SORTED_NUMERIC = 5;
static final byte SORTED_NUMERIC_SINGLETON = 6;
- static final int VERSION_START = 2;
+ static final int VERSION_START = 3;
static final int VERSION_CURRENT = VERSION_START;
+
+ // clone for merge: when merging we don't do any instances.put()s
+ DirectDocValuesProducer(DirectDocValuesProducer original) throws IOException {
+ assert Thread.holdsLock(original);
+ numerics.putAll(original.numerics);
+ binaries.putAll(original.binaries);
+ sorteds.putAll(original.sorteds);
+ sortedSets.putAll(original.sortedSets);
+ sortedNumerics.putAll(original.sortedNumerics);
+ data = original.data.clone();
+
+ numericInstances.putAll(original.numericInstances);
+ binaryInstances.putAll(original.binaryInstances);
+ sortedInstances.putAll(original.sortedInstances);
+ sortedSetInstances.putAll(original.sortedSetInstances);
+ sortedNumericInstances.putAll(original.sortedNumericInstances);
+ docsWithFieldInstances.putAll(original.docsWithFieldInstances);
+
+ numEntries = original.numEntries;
+ maxDoc = original.maxDoc;
+ ramBytesUsed = new AtomicLong(original.ramBytesUsed.get());
+ version = original.version;
+ merging = true;
+ }
DirectDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
maxDoc = state.segmentInfo.getDocCount();
+ merging = false;
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
// read in the entries from the metadata file.
ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass()));
boolean success = false;
try {
- version = CodecUtil.checkHeader(in, metaCodec,
- VERSION_START,
- VERSION_CURRENT);
+ version = CodecUtil.checkSegmentHeader(in, metaCodec, VERSION_START, VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
numEntries = readFields(in, state.fieldInfos);
CodecUtil.checkFooter(in);
@@ -114,9 +140,8 @@ class DirectDocValuesProducer extends Do
this.data = state.directory.openInput(dataName, state.context);
success = false;
try {
- final int version2 = CodecUtil.checkHeader(data, dataCodec,
- VERSION_START,
- VERSION_CURRENT);
+ final int version2 = CodecUtil.checkSegmentHeader(data, dataCodec, VERSION_START, VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
if (version != version2) {
throw new CorruptIndexException("Format versions mismatch: meta=" + version + ", data=" + version2, data);
}
@@ -261,7 +286,10 @@ class DirectDocValuesProducer extends Do
if (instance == null) {
// Lazy load
instance = loadNumeric(numerics.get(field.name));
- numericInstances.put(field.name, instance);
+ if (!merging) {
+ numericInstances.put(field.name, instance);
+ ramBytesUsed.addAndGet(instance.ramBytesUsed());
+ }
}
return instance.numerics;
}
@@ -275,7 +303,6 @@ class DirectDocValuesProducer extends Do
final byte[] values = new byte[entry.count];
data.readBytes(values, 0, entry.count);
ret.bytesUsed = RamUsageEstimator.sizeOf(values);
- ramBytesUsed.addAndGet(ret.bytesUsed);
ret.numerics = new NumericDocValues() {
@Override
public long get(int idx) {
@@ -292,7 +319,6 @@ class DirectDocValuesProducer extends Do
values[i] = data.readShort();
}
ret.bytesUsed = RamUsageEstimator.sizeOf(values);
- ramBytesUsed.addAndGet(ret.bytesUsed);
ret.numerics = new NumericDocValues() {
@Override
public long get(int idx) {
@@ -309,7 +335,6 @@ class DirectDocValuesProducer extends Do
values[i] = data.readInt();
}
ret.bytesUsed = RamUsageEstimator.sizeOf(values);
- ramBytesUsed.addAndGet(ret.bytesUsed);
ret.numerics = new NumericDocValues() {
@Override
public long get(int idx) {
@@ -326,7 +351,6 @@ class DirectDocValuesProducer extends Do
values[i] = data.readLong();
}
ret.bytesUsed = RamUsageEstimator.sizeOf(values);
- ramBytesUsed.addAndGet(ret.bytesUsed);
ret.numerics = new NumericDocValues() {
@Override
public long get(int idx) {
@@ -347,7 +371,10 @@ class DirectDocValuesProducer extends Do
if (instance == null) {
// Lazy load
instance = loadBinary(binaries.get(field.name));
- binaryInstances.put(field.name, instance);
+ if (!merging) {
+ binaryInstances.put(field.name, instance);
+ ramBytesUsed.addAndGet(instance.ramBytesUsed());
+ }
}
final byte[] bytes = instance.bytes;
final int[] address = instance.address;
@@ -377,8 +404,6 @@ class DirectDocValuesProducer extends Do
}
address[entry.count] = data.readInt();
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes) + RamUsageEstimator.sizeOf(address));
-
BinaryRawValues values = new BinaryRawValues();
values.bytes = bytes;
values.address = address;
@@ -394,7 +419,10 @@ class DirectDocValuesProducer extends Do
if (instance == null) {
// Lazy load
instance = loadSorted(field);
- sortedInstances.put(field.name, instance);
+ if (!merging) {
+ sortedInstances.put(field.name, instance);
+ ramBytesUsed.addAndGet(instance.ramBytesUsed());
+ }
}
}
return newSortedInstance(instance.docToOrd.numerics, getBinary(field), entry.values.count);
@@ -439,7 +467,10 @@ class DirectDocValuesProducer extends Do
if (instance == null) {
// Lazy load
instance = loadSortedNumeric(entry);
- sortedNumericInstances.put(field.name, instance);
+ if (!merging) {
+ sortedNumericInstances.put(field.name, instance);
+ ramBytesUsed.addAndGet(instance.ramBytesUsed());
+ }
}
if (entry.docToAddress == null) {
@@ -489,7 +520,10 @@ class DirectDocValuesProducer extends Do
if (instance == null) {
// Lazy load
instance = loadSortedSet(entry);
- sortedSetInstances.put(field.name, instance);
+ if (!merging) {
+ sortedSetInstances.put(field.name, instance);
+ ramBytesUsed.addAndGet(instance.ramBytesUsed());
+ }
}
if (instance.docToOrdAddress == null) {
@@ -573,7 +607,10 @@ class DirectDocValuesProducer extends Do
bits[i] = data.readLong();
}
instance = new FixedBitSet(bits, maxDoc);
- docsWithFieldInstances.put(field.name, instance);
+ if (!merging) {
+ docsWithFieldInstances.put(field.name, instance);
+ ramBytesUsed.addAndGet(instance.ramBytesUsed());
+ }
}
}
return instance;
@@ -601,6 +638,11 @@ class DirectDocValuesProducer extends Do
}
@Override
+ public synchronized DocValuesProducer getMergeInstance() throws IOException {
+ return new DirectDocValuesProducer(this);
+ }
+
+ @Override
public void close() throws IOException {
data.close();
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java Sun Oct 5 14:43:47 2014
@@ -75,7 +75,6 @@ public class FSTOrdTermsReader extends F
static final int INTERVAL = FSTOrdTermsWriter.SKIP_INTERVAL;
final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader;
- int version;
//static final boolean TEST = false;
public FSTOrdTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
@@ -89,11 +88,20 @@ public class FSTOrdTermsReader extends F
try {
indexIn = state.directory.openChecksumInput(termsIndexFileName, state.context);
blockIn = state.directory.openInput(termsBlockFileName, state.context);
- version = readHeader(indexIn);
- readHeader(blockIn);
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(blockIn);
+ int version = CodecUtil.checkSegmentHeader(indexIn, FSTOrdTermsWriter.TERMS_INDEX_CODEC_NAME,
+ FSTOrdTermsWriter.VERSION_START,
+ FSTOrdTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+ int version2 = CodecUtil.checkSegmentHeader(blockIn, FSTOrdTermsWriter.TERMS_CODEC_NAME,
+ FSTOrdTermsWriter.VERSION_START,
+ FSTOrdTermsWriter.VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
+ if (version != version2) {
+ throw new CorruptIndexException("Format versions mismatch: index=" + version + ", terms=" + version2, blockIn);
}
+
+ CodecUtil.checksumEntireFile(blockIn);
this.postingsReader.init(blockIn);
seekDir(blockIn);
@@ -114,11 +122,7 @@ public class FSTOrdTermsReader extends F
TermsReader previous = fields.put(fieldInfo.name, current);
checkFieldSummary(state.segmentInfo, indexIn, blockIn, current, previous);
}
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checkFooter(indexIn);
- } else {
- CodecUtil.checkEOF(indexIn);
- }
+ CodecUtil.checkFooter(indexIn);
success = true;
} finally {
if (success) {
@@ -129,17 +133,8 @@ public class FSTOrdTermsReader extends F
}
}
- private int readHeader(IndexInput in) throws IOException {
- return CodecUtil.checkHeader(in, FSTOrdTermsWriter.TERMS_CODEC_NAME,
- FSTOrdTermsWriter.TERMS_VERSION_START,
- FSTOrdTermsWriter.TERMS_VERSION_CURRENT);
- }
private void seekDir(IndexInput in) throws IOException {
- if (version >= FSTOrdTermsWriter.TERMS_VERSION_CHECKSUM) {
- in.seek(in.length() - CodecUtil.footerLength() - 8);
- } else {
- in.seek(in.length() - 8);
- }
+ in.seek(in.length() - CodecUtil.footerLength() - 8);
in.seek(in.readLong());
}
private void checkFieldSummary(SegmentInfo info, IndexInput indexIn, IndexInput blockIn, TermsReader field, TermsReader previous) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java Sun Oct 5 14:43:47 2014
@@ -39,7 +39,6 @@ import org.apache.lucene.store.RAMOutput
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
@@ -76,7 +75,7 @@ import org.apache.lucene.util.fst.Util;
* <ul>
* <li>TermIndex(.tix) --> Header, TermFST<sup>NumFields</sup>, Footer</li>
* <li>TermFST --> {@link FST FST<long>}</li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
* </ul>
*
@@ -114,7 +113,7 @@ import org.apache.lucene.util.fst.Util;
* <li>StatsBlock --> < DocFreq[Same?], (TotalTermFreq-DocFreq) ? > <sup>NumTerms</sup>
* <li>MetaLongsBlock --> < LongDelta<sup>LongsSize</sup>, BytesSize > <sup>NumTerms</sup>
* <li>MetaBytesBlock --> Byte <sup>MetaBytesBlockLength</sup>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeSegmentHeader CodecHeader}</li>
* <li>DirOffset --> {@link DataOutput#writeLong Uint64}</li>
* <li>NumFields, FieldNumber, DocCount, DocFreq, LongsSize,
* FieldNumber, DocCount --> {@link DataOutput#writeVInt VInt}</li>
@@ -149,10 +148,11 @@ import org.apache.lucene.util.fst.Util;
public class FSTOrdTermsWriter extends FieldsConsumer {
static final String TERMS_INDEX_EXTENSION = "tix";
static final String TERMS_BLOCK_EXTENSION = "tbk";
- static final String TERMS_CODEC_NAME = "FST_ORD_TERMS_DICT";
- public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CHECKSUM = 1;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
+ static final String TERMS_CODEC_NAME = "FSTOrdTerms";
+ static final String TERMS_INDEX_CODEC_NAME = "FSTOrdIndex";
+
+ public static final int VERSION_START = 2;
+ public static final int VERSION_CURRENT = VERSION_START;
public static final int SKIP_INTERVAL = 8;
final PostingsWriterBase postingsWriter;
@@ -174,8 +174,10 @@ public class FSTOrdTermsWriter extends F
try {
this.indexOut = state.directory.createOutput(termsIndexFileName, state.context);
this.blockOut = state.directory.createOutput(termsBlockFileName, state.context);
- writeHeader(indexOut);
- writeHeader(blockOut);
+ CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.writeSegmentHeader(blockOut, TERMS_CODEC_NAME, VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter.init(blockOut);
success = true;
} finally {
@@ -260,9 +262,6 @@ public class FSTOrdTermsWriter extends F
}
}
- private void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
- }
private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
out.writeLong(dirStart);
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java Sun Oct 5 14:43:47 2014
@@ -72,7 +72,6 @@ public class FSTTermsReader extends Fiel
final TreeMap<String, TermsReader> fields = new TreeMap<>();
final PostingsReaderBase postingsReader;
//static boolean TEST = false;
- final int version;
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException {
final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
@@ -82,10 +81,11 @@ public class FSTTermsReader extends Fiel
boolean success = false;
try {
- version = readHeader(in);
- if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
- CodecUtil.checksumEntireFile(in);
- }
+ CodecUtil.checkSegmentHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
+ FSTTermsWriter.TERMS_VERSION_START,
+ FSTTermsWriter.TERMS_VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.checksumEntireFile(in);
this.postingsReader.init(in);
seekDir(in);
@@ -113,17 +113,8 @@ public class FSTTermsReader extends Fiel
}
}
- private int readHeader(IndexInput in) throws IOException {
- return CodecUtil.checkHeader(in, FSTTermsWriter.TERMS_CODEC_NAME,
- FSTTermsWriter.TERMS_VERSION_START,
- FSTTermsWriter.TERMS_VERSION_CURRENT);
- }
private void seekDir(IndexInput in) throws IOException {
- if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
- in.seek(in.length() - CodecUtil.footerLength() - 8);
- } else {
- in.seek(in.length() - 8);
- }
+ in.seek(in.length() - CodecUtil.footerLength() - 8);
in.seek(in.readLong());
}
private void checkFieldSummary(SegmentInfo info, IndexInput in, TermsReader field, TermsReader previous) throws IOException {
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java Sun Oct 5 14:43:47 2014
@@ -39,7 +39,6 @@ import org.apache.lucene.store.RAMOutput
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
@@ -91,7 +90,7 @@ import org.apache.lucene.util.fst.Util;
* <li>TermFST --> {@link FST FST<TermData>}</li>
* <li>TermData --> Flag, BytesSize?, LongDelta<sup>LongsSize</sup>?, Byte<sup>BytesSize</sup>?,
* < DocFreq[Same?], (TotalTermFreq-DocFreq) > ? </li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
+ * <li>Header --> {@link CodecUtil#writeSegmentHeader SegmentHeader}</li>
* <li>DirOffset --> {@link DataOutput#writeLong Uint64}</li>
* <li>DocFreq, LongsSize, BytesSize, NumFields,
* FieldNumber, DocCount --> {@link DataOutput#writeVInt VInt}</li>
@@ -123,10 +122,9 @@ import org.apache.lucene.util.fst.Util;
public class FSTTermsWriter extends FieldsConsumer {
static final String TERMS_EXTENSION = "tmp";
- static final String TERMS_CODEC_NAME = "FST_TERMS_DICT";
- public static final int TERMS_VERSION_START = 0;
- public static final int TERMS_VERSION_CHECKSUM = 1;
- public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_CHECKSUM;
+ static final String TERMS_CODEC_NAME = "FSTTerms";
+ public static final int TERMS_VERSION_START = 2;
+ public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START;
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
@@ -144,7 +142,9 @@ public class FSTTermsWriter extends Fiel
boolean success = false;
try {
- writeHeader(out);
+ CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT,
+ state.segmentInfo.getId(), state.segmentSuffix);
+
this.postingsWriter.init(out);
success = true;
} finally {
@@ -154,10 +154,6 @@ public class FSTTermsWriter extends Fiel
}
}
- private void writeHeader(IndexOutput out) throws IOException {
- CodecUtil.writeHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT);
- }
-
private void writeTrailer(IndexOutput out, long dirStart) throws IOException {
out.writeLong(dirStart);
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java Sun Oct 5 14:43:47 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.MathUtil;
import org.apache.lucene.util.fst.Builder;
@@ -75,10 +74,10 @@ class MemoryDocValuesConsumer extends Do
try {
String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
data = state.directory.createOutput(dataName, state.context);
- CodecUtil.writeHeader(data, dataCodec, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(data, dataCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
meta = state.directory.createOutput(metaName, state.context);
- CodecUtil.writeHeader(meta, metaCodec, VERSION_CURRENT);
+ CodecUtil.writeSegmentHeader(meta, metaCodec, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
success = true;
} finally {
if (!success) {