You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/08/07 00:57:01 UTC
svn commit: r1370055 - in
/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs:
MultiLevelSkipListReader.java MultiLevelSkipListWriter.java
block/BlockSkipReader.java block/BlockSkipWriter.java
Author: mikemccand
Date: Mon Aug 6 22:57:00 2012
New Revision: 1370055
URL: http://svn.apache.org/viewvc?rev=1370055&view=rev
Log:
LUCENE-3892: decouple level 0 skipInterval from level 1+
Modified:
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java?rev=1370055&r1=1370054&r2=1370055&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java Mon Aug 6 22:57:00 2012
@@ -65,24 +65,31 @@ public abstract class MultiLevelSkipList
private long lastChildPointer; // childPointer of last read skip entry with docId <= target
private boolean inputIsBuffered;
-
- public MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) {
+ private final int skipMultiplier;
+
+ // nocommit made protected from public
+ protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval, int skipMultiplier) {
this.skipStream = new IndexInput[maxSkipLevels];
this.skipPointer = new long[maxSkipLevels];
this.childPointer = new long[maxSkipLevels];
this.numSkipped = new int[maxSkipLevels];
this.maxNumberOfSkipLevels = maxSkipLevels;
this.skipInterval = new int[maxSkipLevels];
+ this.skipMultiplier = skipMultiplier;
this.skipStream [0]= skipStream;
this.inputIsBuffered = (skipStream instanceof BufferedIndexInput);
this.skipInterval[0] = skipInterval;
for (int i = 1; i < maxSkipLevels; i++) {
// cache skip intervals
- this.skipInterval[i] = this.skipInterval[i - 1] * skipInterval;
+ this.skipInterval[i] = this.skipInterval[i - 1] * skipMultiplier;
}
skipDoc = new int[maxSkipLevels];
}
+ // skipMultiplier and skipInterval are the same:
+ protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) {
+ this(skipStream, maxSkipLevels, skipInterval, skipInterval);
+ }
/** Returns the id of the doc to which the last call of {@link #skipTo(int)}
* has skipped. */
@@ -157,7 +164,7 @@ public abstract class MultiLevelSkipList
numSkipped[level] = numSkipped[level + 1] - skipInterval[level + 1];
skipDoc[level] = lastDoc;
if (level > 0) {
- childPointer[level] = skipStream[level].readVLong() + skipPointer[level - 1];
+ childPointer[level] = skipStream[level].readVLong() + skipPointer[level - 1];
}
}
@@ -187,7 +194,12 @@ public abstract class MultiLevelSkipList
/** Loads the skip levels */
private void loadSkipLevels() throws IOException {
- numberOfSkipLevels = MathUtil.log(docCount, skipInterval[0]);
+ if (docCount <= skipInterval[0]) {
+ numberOfSkipLevels = 1;
+ } else {
+ numberOfSkipLevels = 1+MathUtil.log(docCount/skipInterval[0], skipMultiplier);
+ }
+
if (numberOfSkipLevels > maxNumberOfSkipLevels) {
numberOfSkipLevels = maxNumberOfSkipLevels;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java?rev=1370055&r1=1370054&r2=1370055&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java Mon Aug 6 22:57:00 2012
@@ -54,15 +54,23 @@ public abstract class MultiLevelSkipList
// the skip interval in the list with level = 0
private int skipInterval;
+
+ // skipInterval used for level > 0
+ private int skipMultiplier;
// for every skip level a different buffer is used
private RAMOutputStream[] skipBuffer;
- protected MultiLevelSkipListWriter(int skipInterval, int maxSkipLevels, int df) {
+ protected MultiLevelSkipListWriter(int skipInterval, int skipMultiplier, int maxSkipLevels, int df) {
this.skipInterval = skipInterval;
+ this.skipMultiplier = skipMultiplier;
// calculate the maximum number of skip levels for this document frequency
- numberOfSkipLevels = MathUtil.log(df, skipInterval);
+ if (df <= skipInterval) {
+ numberOfSkipLevels = 1;
+ } else {
+ numberOfSkipLevels = 1+MathUtil.log(df/skipInterval, skipMultiplier);
+ }
// make sure it does not exceed maxSkipLevels
if (numberOfSkipLevels > maxSkipLevels) {
@@ -70,6 +78,11 @@ public abstract class MultiLevelSkipList
}
}
+ // skipMultiplier and skipInterval are the same:
+ protected MultiLevelSkipListWriter(int skipInterval, int maxSkipLevels, int df) {
+ this(skipInterval, skipInterval, maxSkipLevels, df);
+ }
+
protected void init() {
skipBuffer = new RAMOutputStream[numberOfSkipLevels];
for (int i = 0; i < numberOfSkipLevels; i++) {
@@ -95,7 +108,7 @@ public abstract class MultiLevelSkipList
* @param skipBuffer the skip buffer to write to
*/
protected abstract void writeSkipData(int level, IndexOutput skipBuffer) throws IOException;
-
+
/**
* Writes the current skip data to the buffers. The current document frequency determines
* the max level is skip data is to be written to.
@@ -104,11 +117,15 @@ public abstract class MultiLevelSkipList
* @throws IOException
*/
public void bufferSkip(int df) throws IOException {
- int numLevels;
+
+ assert df % skipInterval == 0;
+ int numLevels = 1;
+ df /= skipInterval;
// determine max level
- for (numLevels = 0; (df % skipInterval) == 0 && numLevels < numberOfSkipLevels; df /= skipInterval) {
+ while ((df % skipMultiplier) == 0 && numLevels < numberOfSkipLevels) {
numLevels++;
+ df /= skipMultiplier;
}
long childPointer = 0;
@@ -150,5 +167,4 @@ public abstract class MultiLevelSkipList
return skipPointer;
}
-
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java?rev=1370055&r1=1370054&r2=1370055&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java Mon Aug 6 22:57:00 2012
@@ -69,7 +69,7 @@ final class BlockSkipReader extends Mult
private int lastPosBufferUpto;
public BlockSkipReader(IndexInput skipStream, int maxSkipLevels, int blockSize, boolean hasPos, boolean hasOffsets, boolean hasPayloads) {
- super(skipStream, maxSkipLevels, blockSize);
+ super(skipStream, maxSkipLevels, blockSize, 4);
this.blockSize = blockSize;
docPointer = new long[maxSkipLevels];
if (hasPos) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java?rev=1370055&r1=1370054&r2=1370055&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java Mon Aug 6 22:57:00 2012
@@ -72,7 +72,9 @@ final class BlockSkipWriter extends Mult
private boolean fieldHasPayloads;
public BlockSkipWriter(int maxSkipLevels, int blockSize, int docCount, IndexOutput docOut, IndexOutput posOut, IndexOutput payOut) {
- super(blockSize, maxSkipLevels, docCount);
+ // nocommit figure out what skipMultiplier is best (4 is
+ // total guess):
+ super(blockSize, 4, maxSkipLevels, docCount);
this.docOut = docOut;
this.posOut = posOut;
this.payOut = payOut;