You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/01/21 17:24:08 UTC
svn commit: r1653580 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/lucene50/
Author: mikemccand
Date: Wed Jan 21 16:24:08 2015
New Revision: 1653580
URL: http://svn.apache.org/r1653580
Log:
LUCENE-6192: don't overflow int when writing skip data for high freq terms in extremely large indices
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (props changed)
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java?rev=1653580&r1=1653579&r2=1653580&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java Wed Jan 21 16:24:08 2015
@@ -173,13 +173,13 @@ final class Lucene41SkipReader extends M
// if (DEBUG) {
// System.out.println(" delta=" + delta);
// }
- docPointer[level] += skipStream.readVInt();
+ docPointer[level] += skipStream.readVLong();
// if (DEBUG) {
// System.out.println(" docFP=" + docPointer[level]);
// }
if (posPointer != null) {
- posPointer[level] += skipStream.readVInt();
+ posPointer[level] += skipStream.readVLong();
// if (DEBUG) {
// System.out.println(" posFP=" + posPointer[level]);
// }
@@ -193,7 +193,7 @@ final class Lucene41SkipReader extends M
}
if (payPointer != null) {
- payPointer[level] += skipStream.readVInt();
+ payPointer[level] += skipStream.readVLong();
}
}
return delta;
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java?rev=1653580&r1=1653579&r2=1653580&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java Wed Jan 21 16:24:08 2015
@@ -179,10 +179,10 @@ final class Lucene50SkipReader extends M
@Override
protected int readSkipData(int level, IndexInput skipStream) throws IOException {
int delta = skipStream.readVInt();
- docPointer[level] += skipStream.readVInt();
+ docPointer[level] += skipStream.readVLong();
if (posPointer != null) {
- posPointer[level] += skipStream.readVInt();
+ posPointer[level] += skipStream.readVLong();
posBufferUpto[level] = skipStream.readVInt();
if (payloadByteUpto != null) {
@@ -190,7 +190,7 @@ final class Lucene50SkipReader extends M
}
if (payPointer != null) {
- payPointer[level] += skipStream.readVInt();
+ payPointer[level] += skipStream.readVLong();
}
}
return delta;
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java?rev=1653580&r1=1653579&r2=1653580&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java Wed Jan 21 16:24:08 2015
@@ -147,12 +147,12 @@ final class Lucene50SkipWriter extends M
skipBuffer.writeVInt(delta);
lastSkipDoc[level] = curDoc;
- skipBuffer.writeVInt((int) (curDocPointer - lastSkipDocPointer[level]));
+ skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);
lastSkipDocPointer[level] = curDocPointer;
if (fieldHasPositions) {
- skipBuffer.writeVInt((int) (curPosPointer - lastSkipPosPointer[level]));
+ skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);
lastSkipPosPointer[level] = curPosPointer;
skipBuffer.writeVInt(curPosBufferUpto);
@@ -161,7 +161,7 @@ final class Lucene50SkipWriter extends M
}
if (fieldHasOffsets || fieldHasPayloads) {
- skipBuffer.writeVInt((int) (curPayPointer - lastSkipPayPointer[level]));
+ skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);
lastSkipPayPointer[level] = curPayPointer;
}
}
Re: svn commit: r1653580 - in /lucene/dev/branches/branch_5x: ./
lucene/ lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/lucene50/
Posted by Michael McCandless <lu...@mikemccandless.com>.
Thanks Rob, I fixed Lucene41 too.
Mike McCandless
http://blog.mikemccandless.com
On Wed, Jan 21, 2015 at 11:29 AM, Robert Muir <rc...@gmail.com> wrote:
> I think we should also fix Lucene41SkipWriter (src/test) ?
>
> On Wed, Jan 21, 2015 at 11:24 AM, <mi...@apache.org> wrote:
>> Author: mikemccand
>> Date: Wed Jan 21 16:24:08 2015
>> New Revision: 1653580
>>
>> URL: http://svn.apache.org/r1653580
>> Log:
>> LUCENE-6192: don't overflow int when writing skip data for high freq terms in extremely large indices
>>
>> Modified:
>> lucene/dev/branches/branch_5x/ (props changed)
>> lucene/dev/branches/branch_5x/lucene/ (props changed)
>> lucene/dev/branches/branch_5x/lucene/CHANGES.txt (props changed)
>> lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java
>> lucene/dev/branches/branch_5x/lucene/core/ (props changed)
>> lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
>> lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
>>
>> Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java?rev=1653580&r1=1653579&r2=1653580&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java (original)
>> +++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java Wed Jan 21 16:24:08 2015
>> @@ -173,13 +173,13 @@ final class Lucene41SkipReader extends M
>> // if (DEBUG) {
>> // System.out.println(" delta=" + delta);
>> // }
>> - docPointer[level] += skipStream.readVInt();
>> + docPointer[level] += skipStream.readVLong();
>> // if (DEBUG) {
>> // System.out.println(" docFP=" + docPointer[level]);
>> // }
>>
>> if (posPointer != null) {
>> - posPointer[level] += skipStream.readVInt();
>> + posPointer[level] += skipStream.readVLong();
>> // if (DEBUG) {
>> // System.out.println(" posFP=" + posPointer[level]);
>> // }
>> @@ -193,7 +193,7 @@ final class Lucene41SkipReader extends M
>> }
>>
>> if (payPointer != null) {
>> - payPointer[level] += skipStream.readVInt();
>> + payPointer[level] += skipStream.readVLong();
>> }
>> }
>> return delta;
>>
>> Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java?rev=1653580&r1=1653579&r2=1653580&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java (original)
>> +++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java Wed Jan 21 16:24:08 2015
>> @@ -179,10 +179,10 @@ final class Lucene50SkipReader extends M
>> @Override
>> protected int readSkipData(int level, IndexInput skipStream) throws IOException {
>> int delta = skipStream.readVInt();
>> - docPointer[level] += skipStream.readVInt();
>> + docPointer[level] += skipStream.readVLong();
>>
>> if (posPointer != null) {
>> - posPointer[level] += skipStream.readVInt();
>> + posPointer[level] += skipStream.readVLong();
>> posBufferUpto[level] = skipStream.readVInt();
>>
>> if (payloadByteUpto != null) {
>> @@ -190,7 +190,7 @@ final class Lucene50SkipReader extends M
>> }
>>
>> if (payPointer != null) {
>> - payPointer[level] += skipStream.readVInt();
>> + payPointer[level] += skipStream.readVLong();
>> }
>> }
>> return delta;
>>
>> Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java?rev=1653580&r1=1653579&r2=1653580&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java (original)
>> +++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java Wed Jan 21 16:24:08 2015
>> @@ -147,12 +147,12 @@ final class Lucene50SkipWriter extends M
>> skipBuffer.writeVInt(delta);
>> lastSkipDoc[level] = curDoc;
>>
>> - skipBuffer.writeVInt((int) (curDocPointer - lastSkipDocPointer[level]));
>> + skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);
>> lastSkipDocPointer[level] = curDocPointer;
>>
>> if (fieldHasPositions) {
>>
>> - skipBuffer.writeVInt((int) (curPosPointer - lastSkipPosPointer[level]));
>> + skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);
>> lastSkipPosPointer[level] = curPosPointer;
>> skipBuffer.writeVInt(curPosBufferUpto);
>>
>> @@ -161,7 +161,7 @@ final class Lucene50SkipWriter extends M
>> }
>>
>> if (fieldHasOffsets || fieldHasPayloads) {
>> - skipBuffer.writeVInt((int) (curPayPointer - lastSkipPayPointer[level]));
>> + skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);
>> lastSkipPayPointer[level] = curPayPointer;
>> }
>> }
>>
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1653580 - in /lucene/dev/branches/branch_5x: ./
lucene/ lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/lucene50/
Posted by Robert Muir <rc...@gmail.com>.
I think we should also fix Lucene41SkipWriter (src/test) ?
On Wed, Jan 21, 2015 at 11:24 AM, <mi...@apache.org> wrote:
> Author: mikemccand
> Date: Wed Jan 21 16:24:08 2015
> New Revision: 1653580
>
> URL: http://svn.apache.org/r1653580
> Log:
> LUCENE-6192: don't overflow int when writing skip data for high freq terms in extremely large indices
>
> Modified:
> lucene/dev/branches/branch_5x/ (props changed)
> lucene/dev/branches/branch_5x/lucene/ (props changed)
> lucene/dev/branches/branch_5x/lucene/CHANGES.txt (props changed)
> lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java
> lucene/dev/branches/branch_5x/lucene/core/ (props changed)
> lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
> lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
>
> Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java?rev=1653580&r1=1653579&r2=1653580&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java (original)
> +++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipReader.java Wed Jan 21 16:24:08 2015
> @@ -173,13 +173,13 @@ final class Lucene41SkipReader extends M
> // if (DEBUG) {
> // System.out.println(" delta=" + delta);
> // }
> - docPointer[level] += skipStream.readVInt();
> + docPointer[level] += skipStream.readVLong();
> // if (DEBUG) {
> // System.out.println(" docFP=" + docPointer[level]);
> // }
>
> if (posPointer != null) {
> - posPointer[level] += skipStream.readVInt();
> + posPointer[level] += skipStream.readVLong();
> // if (DEBUG) {
> // System.out.println(" posFP=" + posPointer[level]);
> // }
> @@ -193,7 +193,7 @@ final class Lucene41SkipReader extends M
> }
>
> if (payPointer != null) {
> - payPointer[level] += skipStream.readVInt();
> + payPointer[level] += skipStream.readVLong();
> }
> }
> return delta;
>
> Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java?rev=1653580&r1=1653579&r2=1653580&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java (original)
> +++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java Wed Jan 21 16:24:08 2015
> @@ -179,10 +179,10 @@ final class Lucene50SkipReader extends M
> @Override
> protected int readSkipData(int level, IndexInput skipStream) throws IOException {
> int delta = skipStream.readVInt();
> - docPointer[level] += skipStream.readVInt();
> + docPointer[level] += skipStream.readVLong();
>
> if (posPointer != null) {
> - posPointer[level] += skipStream.readVInt();
> + posPointer[level] += skipStream.readVLong();
> posBufferUpto[level] = skipStream.readVInt();
>
> if (payloadByteUpto != null) {
> @@ -190,7 +190,7 @@ final class Lucene50SkipReader extends M
> }
>
> if (payPointer != null) {
> - payPointer[level] += skipStream.readVInt();
> + payPointer[level] += skipStream.readVLong();
> }
> }
> return delta;
>
> Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java?rev=1653580&r1=1653579&r2=1653580&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java (original)
> +++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java Wed Jan 21 16:24:08 2015
> @@ -147,12 +147,12 @@ final class Lucene50SkipWriter extends M
> skipBuffer.writeVInt(delta);
> lastSkipDoc[level] = curDoc;
>
> - skipBuffer.writeVInt((int) (curDocPointer - lastSkipDocPointer[level]));
> + skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]);
> lastSkipDocPointer[level] = curDocPointer;
>
> if (fieldHasPositions) {
>
> - skipBuffer.writeVInt((int) (curPosPointer - lastSkipPosPointer[level]));
> + skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]);
> lastSkipPosPointer[level] = curPosPointer;
> skipBuffer.writeVInt(curPosBufferUpto);
>
> @@ -161,7 +161,7 @@ final class Lucene50SkipWriter extends M
> }
>
> if (fieldHasOffsets || fieldHasPayloads) {
> - skipBuffer.writeVInt((int) (curPayPointer - lastSkipPayPointer[level]));
> + skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]);
> lastSkipPayPointer[level] = curPayPointer;
> }
> }
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org