You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/08/10 18:29:44 UTC
svn commit: r1371762 - in
/lucene/dev/branches/pforcodec_3892/lucene/core/src:
java/org/apache/lucene/codecs/block/ test/org/apache/lucene/index/
Author: mikemccand
Date: Fri Aug 10 16:29:44 2012
New Revision: 1371762
URL: http://svn.apache.org/viewvc?rev=1371762&view=rev
Log:
LUCENE-3892: resolve some of the nocommits...
Modified:
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/ForUtil.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java?rev=1371762&r1=1371761&r2=1371762&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsFormat.java Fri Aug 10 16:29:44 2012
@@ -42,9 +42,7 @@ public final class BlockPostingsFormat e
private final int minTermBlockSize;
private final int maxTermBlockSize;
- // nocommit is this right?:
- // NOTE: should be at least 64 because of PackedInts long-aligned encoding/decoding
- // NOTE: must be factor of ... 64?
+ // NOTE: must be factor of 64 because of PackedInts long-aligned encoding/decoding
public final static int BLOCK_SIZE = 128;
public BlockPostingsFormat() {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java?rev=1371762&r1=1371761&r2=1371762&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java Fri Aug 10 16:29:44 2012
@@ -62,12 +62,8 @@ public final class BlockPostingsReader e
public static boolean DEBUG = false;
- // nocommit
- final String segment;
-
public BlockPostingsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo segmentInfo, IOContext ioContext, String segmentSuffix) throws IOException {
boolean success = false;
- segment = segmentInfo.name;
IndexInput docIn = null;
IndexInput posIn = null;
IndexInput payIn = null;
@@ -376,7 +372,7 @@ public final class BlockPostingsReader e
public DocsEnum reset(Bits liveDocs, IntBlockTermState termState) throws IOException {
this.liveDocs = liveDocs;
if (DEBUG) {
- System.out.println(" FPR.reset: seg=" + segment + " termState=" + termState);
+ System.out.println(" FPR.reset: termState=" + termState);
}
docFreq = termState.docFreq;
docTermStartFP = termState.docStartFP;
@@ -475,7 +471,7 @@ public final class BlockPostingsReader e
@Override
public int advance(int target) throws IOException {
- // nocommit make frq block load lazy/skippable
+ // TODO: make frq block load lazy/skippable
if (DEBUG) {
System.out.println(" FPR.advance target=" + target);
}
@@ -781,7 +777,7 @@ public final class BlockPostingsReader e
@Override
public int advance(int target) throws IOException {
- // nocommit make frq block load lazy/skippable
+ // TODO: make frq block load lazy/skippable
if (DEBUG) {
System.out.println(" FPR.advance target=" + target);
}
@@ -878,7 +874,7 @@ public final class BlockPostingsReader e
}
}
- // nocommit in theory we could avoid loading frq block
+ // TODO: in theory we could avoid loading frq block
// when not needed, ie, use skip data to load how far to
// seek the pos pointe ... instead of having to load frq
// blocks only to sum up how many positions to skip
@@ -1272,7 +1268,7 @@ public final class BlockPostingsReader e
@Override
public int advance(int target) throws IOException {
- // nocommit make frq block load lazy/skippable
+ // TODO: make frq block load lazy/skippable
if (DEBUG) {
System.out.println(" FPR.advance target=" + target);
}
@@ -1375,7 +1371,7 @@ public final class BlockPostingsReader e
}
}
- // nocommit in theory we could avoid loading frq block
+ // TODO: in theory we could avoid loading frq block
// when not needed, ie, use skip data to load how far to
// seek the pos pointe ... instead of having to load frq
// blocks only to sum up how many positions to skip
@@ -1393,9 +1389,6 @@ public final class BlockPostingsReader e
if (indexHasPayloads) {
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
- if (indexHasOffsets) {
- lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
- }
posBufferUpto++;
}
if (DEBUG) {
@@ -1420,13 +1413,8 @@ public final class BlockPostingsReader e
}
if (indexHasOffsets) {
- // Must load offset blocks merely to sum
- // up into lastStartOffset:
- forUtil.readBlock(payIn, encoded, offsetStartDeltaBuffer);
- forUtil.readBlock(payIn, encoded, offsetLengthBuffer);
- for(int i=0;i<BLOCK_SIZE;i++) {
- lastStartOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
- }
+ forUtil.skipBlock(payIn);
+ forUtil.skipBlock(payIn);
}
toSkip -= BLOCK_SIZE;
}
@@ -1437,9 +1425,6 @@ public final class BlockPostingsReader e
if (indexHasPayloads) {
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
- if (indexHasOffsets) {
- lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
- }
posBufferUpto++;
}
if (DEBUG) {
@@ -1449,8 +1434,6 @@ public final class BlockPostingsReader e
position = 0;
payloadLength = 0;
- // nocommit why carefully sum up lastStartOffset above
- // only to set it to 0 now?
lastStartOffset = 0;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java?rev=1371762&r1=1371761&r2=1371762&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java Fri Aug 10 16:29:44 2012
@@ -53,8 +53,6 @@ import org.apache.lucene.util.packed.Pac
*/
public final class BlockPostingsWriter extends PostingsWriterBase {
- // nocommit move these constants to the PF:
-
static final int maxSkipLevels = 10;
final static String TERMS_CODEC = "BlockPostingsWriterTerms";
@@ -170,7 +168,7 @@ public final class BlockPostingsWriter e
docDeltaBuffer = new int[MAX_DATA_SIZE];
freqBuffer = new int[MAX_DATA_SIZE];
- // nocommit should we try skipping every 2/4 blocks...?
+ // TODO: should we try skipping every 2/4 blocks...?
skipWriter = new BlockSkipWriter(maxSkipLevels,
BLOCK_SIZE,
state.segmentInfo.getDocCount(),
@@ -418,7 +416,7 @@ public final class BlockPostingsWriter e
if (posBufferUpto > 0) {
posOut.writeVInt(posBufferUpto);
- // nocommit should we send offsets/payloads to
+ // TODO: should we send offsets/payloads to
// .pay...? seems wasteful (have to store extra
// vLong for low (< BLOCK_SIZE) DF terms = vast vast
// majority)
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/ForUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/ForUtil.java?rev=1371762&r1=1371761&r2=1371762&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/ForUtil.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/ForUtil.java Fri Aug 10 16:29:44 2012
@@ -214,7 +214,6 @@ final class ForUtil {
in.seek(in.getFilePointer() + encodedSize);
}
- // nocommit: we must have a util function for this, hmm?
private static boolean isAllEqual(final int[] data) {
final long v = data[0];
for (int i = 1; i < BLOCK_SIZE; ++i) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java?rev=1371762&r1=1371761&r2=1371762&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java Fri Aug 10 16:29:44 2012
@@ -908,8 +908,8 @@ public class TestPostingsFormat extends
// NOTE: you can also test "weaker" index options than
// you indexed with:
- //testTerms(fieldsProducer, EnumSet.allOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
- testTerms(fieldsProducer, EnumSet.complementOf(EnumSet.of(Option.THREADS)), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ testTerms(fieldsProducer, EnumSet.allOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ //testTerms(fieldsProducer, EnumSet.complementOf(EnumSet.of(Option.THREADS)), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
fieldsProducer.close();
dir.close();