You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/08/27 13:33:30 UTC
svn commit: r1517792 - in /lucene/dev/branches/lucene3069/lucene:
codecs/src/java/org/apache/lucene/codecs/blockterms/
codecs/src/java/org/apache/lucene/codecs/pulsing/
core/src/java/org/apache/lucene/codecs/
core/src/java/org/apache/lucene/codecs/luce...
Author: han
Date: Tue Aug 27 11:33:29 2013
New Revision: 1517792
URL: http://svn.apache.org/r1517792
Log:
LUCENE-3069: add version check for impersonation
Modified:
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java
lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java?rev=1517792&r1=1517791&r2=1517792&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java Tue Aug 27 11:33:29 2013
@@ -142,7 +142,7 @@ public class BlockTermsReader extends Fi
final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
final long sumDocFreq = in.readVLong();
final int docCount = in.readVInt();
- final int longsSize = in.readVInt();
+ final int longsSize = version >= BlockTermsWriter.VERSION_META_ARRAY ? in.readVInt() : 0;
if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")");
}
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java?rev=1517792&r1=1517791&r2=1517792&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java Tue Aug 27 11:33:29 2013
@@ -60,7 +60,9 @@ public class BlockTermsWriter extends Fi
// Initial format
public static final int VERSION_START = 0;
public static final int VERSION_APPEND_ONLY = 1;
+ public static final int VERSION_META_ARRAY = 2;
public static final int VERSION_CURRENT = VERSION_APPEND_ONLY;
+ //public static final int VERSION_CURRENT = VERSION_META_ARRAY;
/** Extension of terms file */
static final String TERMS_EXTENSION = "tib";
@@ -136,9 +138,7 @@ public class BlockTermsWriter extends Fi
@Override
public void close() throws IOException {
-
try {
-
final long dirStart = out.getFilePointer();
out.writeVInt(fields.size());
@@ -151,7 +151,9 @@ public class BlockTermsWriter extends Fi
}
out.writeVLong(field.sumDocFreq);
out.writeVInt(field.docCount);
- out.writeVInt(field.longsSize);
+ if (VERSION_CURRENT >= VERSION_META_ARRAY) {
+ out.writeVInt(field.longsSize);
+ }
}
writeTrailer(dirStart);
} finally {
Modified: lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java?rev=1517792&r1=1517791&r2=1517792&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsWriter.java Tue Aug 27 11:33:29 2013
@@ -63,7 +63,8 @@ public final class PulsingPostingsWriter
final static int VERSION_META_ARRAY = 1;
- final static int VERSION_CURRENT = VERSION_META_ARRAY;
+ //final static int VERSION_CURRENT = VERSION_META_ARRAY;
+ final static int VERSION_CURRENT = VERSION_START;
private SegmentWriteState segmentState;
private IndexOutput termsOut;
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java?rev=1517792&r1=1517791&r2=1517792&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java Tue Aug 27 11:33:29 2013
@@ -158,7 +158,7 @@ public class BlockTreeTermsReader extend
final long sumTotalTermFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : in.readVLong();
final long sumDocFreq = in.readVLong();
final int docCount = in.readVInt();
- final int longsSize = in.readVInt();
+ final int longsSize = version >= BlockTreeTermsWriter.TERMS_VERSION_META_ARRAY ? in.readVInt() : 0;
if (docCount < 0 || docCount > info.getDocCount()) { // #docs with field must be <= #docs
throw new CorruptIndexException("invalid docCount: " + docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")");
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java?rev=1517792&r1=1517791&r2=1517792&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java Tue Aug 27 11:33:29 2013
@@ -205,6 +205,9 @@ public class BlockTreeTermsWriter extend
/** Append-only */
public static final int TERMS_VERSION_APPEND_ONLY = 1;
+ /** Meta data as array */
+ public static final int TERMS_VERSION_META_ARRAY = 2;
+
/** Current terms format. */
public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_APPEND_ONLY;
@@ -218,6 +221,9 @@ public class BlockTreeTermsWriter extend
/** Append-only */
public static final int TERMS_INDEX_VERSION_APPEND_ONLY = 1;
+ /** Meta data as array */
+ public static final int TERMS_INDEX_VERSION_META_ARRAY = 2;
+
/** Current index format. */
public static final int TERMS_INDEX_VERSION_CURRENT = TERMS_INDEX_VERSION_APPEND_ONLY;
@@ -1121,7 +1127,9 @@ public class BlockTreeTermsWriter extend
}
out.writeVLong(field.sumDocFreq);
out.writeVInt(field.docCount);
- out.writeVInt(field.longsSize);
+ if (TERMS_VERSION_CURRENT >= TERMS_VERSION_META_ARRAY) {
+ out.writeVInt(field.longsSize);
+ }
indexOut.writeVLong(field.indexStartFP);
}
writeTrailer(out, dirStart);
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java?rev=1517792&r1=1517791&r2=1517792&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java Tue Aug 27 11:33:29 2013
@@ -59,6 +59,7 @@ public final class Lucene41PostingsReade
private final IndexInput payIn;
private final ForUtil forUtil;
+ private int version;
// public static boolean DEBUG = false;
@@ -71,27 +72,21 @@ public final class Lucene41PostingsReade
try {
docIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene41PostingsFormat.DOC_EXTENSION),
ioContext);
- CodecUtil.checkHeader(docIn,
+ version = CodecUtil.checkHeader(docIn,
Lucene41PostingsWriter.DOC_CODEC,
- Lucene41PostingsWriter.VERSION_CURRENT,
+ Lucene41PostingsWriter.VERSION_START,
Lucene41PostingsWriter.VERSION_CURRENT);
forUtil = new ForUtil(docIn);
if (fieldInfos.hasProx()) {
posIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene41PostingsFormat.POS_EXTENSION),
ioContext);
- CodecUtil.checkHeader(posIn,
- Lucene41PostingsWriter.POS_CODEC,
- Lucene41PostingsWriter.VERSION_CURRENT,
- Lucene41PostingsWriter.VERSION_CURRENT);
+ CodecUtil.checkHeader(posIn, Lucene41PostingsWriter.POS_CODEC, version, version);
if (fieldInfos.hasPayloads() || fieldInfos.hasOffsets()) {
payIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene41PostingsFormat.PAY_EXTENSION),
ioContext);
- CodecUtil.checkHeader(payIn,
- Lucene41PostingsWriter.PAY_CODEC,
- Lucene41PostingsWriter.VERSION_CURRENT,
- Lucene41PostingsWriter.VERSION_CURRENT);
+ CodecUtil.checkHeader(payIn, Lucene41PostingsWriter.PAY_CODEC, version, version);
}
}
@@ -111,7 +106,7 @@ public final class Lucene41PostingsReade
// Make sure we are talking to the matching postings writer
CodecUtil.checkHeader(termsIn,
Lucene41PostingsWriter.TERMS_CODEC,
- Lucene41PostingsWriter.VERSION_CURRENT,
+ Lucene41PostingsWriter.VERSION_START,
Lucene41PostingsWriter.VERSION_CURRENT);
final int indexBlockSize = termsIn.readVInt();
if (indexBlockSize != BLOCK_SIZE) {
@@ -201,6 +196,10 @@ public final class Lucene41PostingsReade
termState.posStartFP = 0;
termState.payStartFP = 0;
}
+ if (version < Lucene41PostingsWriter.VERSION_META_ARRAY) { // impersonation
+ _decodeTerm(in, fieldInfo, termState);
+ return;
+ }
termState.docStartFP += longs[0];
if (fieldHasPositions) {
termState.posStartFP += longs[1];
@@ -220,13 +219,41 @@ public final class Lucene41PostingsReade
termState.lastPosBlockOffset = -1;
}
}
-
if (termState.docFreq > BLOCK_SIZE) {
termState.skipOffset = in.readVLong();
} else {
termState.skipOffset = -1;
}
}
+ private void _decodeTerm(DataInput in, FieldInfo fieldInfo, IntBlockTermState termState) throws IOException {
+ final boolean fieldHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ final boolean fieldHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ final boolean fieldHasPayloads = fieldInfo.hasPayloads();
+ if (termState.docFreq == 1) {
+ termState.singletonDocID = in.readVInt();
+ } else {
+ termState.singletonDocID = -1;
+ termState.docStartFP += in.readVLong();
+ System.out.println(termState.docStartFP);
+ }
+ if (fieldHasPositions) {
+ termState.posStartFP += in.readVLong();
+ if (termState.totalTermFreq > BLOCK_SIZE) {
+ termState.lastPosBlockOffset = in.readVLong();
+ } else {
+ termState.lastPosBlockOffset = -1;
+ }
+ if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) {
+ termState.payStartFP += in.readVLong();
+ }
+ }
+ if (termState.docFreq > BLOCK_SIZE) {
+ termState.skipOffset = in.readVLong();
+ } else {
+ termState.skipOffset = -1;
+ }
+ //System.out.println("PR: state=" + termState);
+ }
@Override
public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java?rev=1517792&r1=1517791&r2=1517792&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java Tue Aug 27 11:33:29 2013
@@ -66,13 +66,14 @@ public final class Lucene41PostingsWrite
// Increment version to change it
final static int VERSION_START = 0;
+ final static int VERSION_META_ARRAY = 1;
final static int VERSION_CURRENT = VERSION_START;
+ //final static int VERSION_CURRENT = VERSION_META_ARRAY;
final IndexOutput docOut;
final IndexOutput posOut;
final IndexOutput payOut;
- final static IntBlockTermState emptyState = new IntBlockTermState();
IntBlockTermState lastState;
// How current field indexes postings:
@@ -224,7 +225,10 @@ public final class Lucene41PostingsWrite
fieldHasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
fieldHasPayloads = fieldInfo.hasPayloads();
skipWriter.setField(fieldHasPositions, fieldHasOffsets, fieldHasPayloads);
- lastState = emptyState;
+ lastState = newTermState();
+ if (VERSION_CURRENT < VERSION_META_ARRAY) {
+ return 0;
+ }
if (fieldHasPositions) {
if (fieldHasPayloads || fieldHasOffsets) {
return 3; // doc + pos + pay FP
@@ -517,12 +521,16 @@ public final class Lucene41PostingsWrite
// System.out.println(" no skip: docCount=" + docCount);
// }
}
+ if (state.totalTermFreq >= BLOCK_SIZE) {
+ state.payTermStartFP = payTermStartFP;
+ } else {
+ state.payTermStartFP = -1;
+ }
// if (DEBUG) {
// System.out.println(" payStartFP=" + payStartFP);
// }
state.docTermStartFP = docTermStartFP;
state.posTermStartFP = posTermStartFP;
- state.payTermStartFP = payTermStartFP;
state.singletonDocID = singletonDocID;
state.skipOffset = skipOffset;
state.lastPosBlockOffset = lastPosBlockOffset;
@@ -538,13 +546,23 @@ public final class Lucene41PostingsWrite
public void encodeTerm(long[] longs, DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
IntBlockTermState state = (IntBlockTermState)_state;
if (absolute) {
- lastState = emptyState;
+ lastState = newTermState();
+ }
+ //System.out.println("PW: state=" + state);
+ //System.out.println(" last=" + lastState);
+ if (VERSION_CURRENT < VERSION_META_ARRAY) { // impersonation
+ _encodeTerm(out, fieldInfo, state);
+ return;
}
longs[0] = state.docTermStartFP - lastState.docTermStartFP;
if (fieldHasPositions) {
longs[1] = state.posTermStartFP - lastState.posTermStartFP;
if (fieldHasPayloads || fieldHasOffsets) {
- longs[2] = state.payTermStartFP - lastState.payTermStartFP;
+ if (state.payTermStartFP == -1) {
+ longs[2] = 0;
+ } else {
+ longs[2] = state.payTermStartFP - lastState.payTermStartFP;
+ }
}
}
if (state.singletonDocID != -1) {
@@ -558,8 +576,33 @@ public final class Lucene41PostingsWrite
if (state.skipOffset != -1) {
out.writeVLong(state.skipOffset);
}
+ if (state.payTermStartFP == -1) {
+ state.payTermStartFP = lastState.payTermStartFP;
+ }
lastState = state;
}
+ public void _encodeTerm(DataOutput out, FieldInfo fieldInfo, IntBlockTermState state) throws IOException {
+ if (state.singletonDocID == -1) {
+ out.writeVLong(state.docTermStartFP - lastState.docTermStartFP);
+ lastState.docTermStartFP = state.docTermStartFP;
+ } else {
+ out.writeVInt(state.singletonDocID);
+ }
+ if (fieldHasPositions) {
+ out.writeVLong(state.posTermStartFP - lastState.posTermStartFP);
+ lastState.posTermStartFP = state.posTermStartFP;
+ if (state.lastPosBlockOffset != -1) {
+ out.writeVLong(state.lastPosBlockOffset);
+ }
+ if ((fieldHasPayloads || fieldHasOffsets) && state.payTermStartFP != -1) {
+ out.writeVLong(state.payTermStartFP - lastState.payTermStartFP);
+ lastState.payTermStartFP = state.payTermStartFP;
+ }
+ }
+ if (state.skipOffset != -1) {
+ out.writeVLong(state.skipOffset);
+ }
+ }
@Override
public void close() throws IOException {