You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/01/25 12:21:35 UTC
svn commit: r1438455 - in /lucene/dev/branches/lucene4547/lucene/core/src:
java/org/apache/lucene/util/packed/ test/org/apache/lucene/util/packed/
Author: jpountz
Date: Fri Jan 25 11:21:35 2013
New Revision: 1438455
URL: http://svn.apache.org/viewvc?rev=1438455&view=rev
Log:
Fix integer overflow.
Modified:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java?rev=1438455&r1=1438454&r2=1438455&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/AbstractBlockPackedWriter.java Fri Jan 25 11:21:35 2013
@@ -96,6 +96,20 @@ abstract class AbstractBlockPackedWriter
++ord;
}
+ // For testing only
+ void addBlockOfZeros() throws IOException {
+ checkNotFinished();
+ if (off != 0 && off != values.length) {
+ throw new IllegalStateException("" + off);
+ }
+ if (off == values.length) {
+ flush();
+ }
+ Arrays.fill(values, 0);
+ off = values.length;
+ ord += values.length;
+ }
+
/** Flush all buffered data to disk. This instance is not usable anymore
* after this method has been called until {@link #reset(DataOutput)} has
* been called. */
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java?rev=1438455&r1=1438454&r2=1438455&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java Fri Jan 25 11:21:35 2013
@@ -45,7 +45,7 @@ public final class BlockPackedReader {
blockShift = Integer.numberOfTrailingZeros(blockSize);
blockMask = blockSize - 1;
final int numBlocks = (int) (valueCount / blockSize) + (valueCount % blockSize == 0 ? 0 : 1);
- if (numBlocks * blockSize < valueCount) {
+ if ((long) numBlocks * blockSize < valueCount) {
throw new IllegalArgumentException("valueCount is too large for this block size");
}
long[] minValues = null;
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java?rev=1438455&r1=1438454&r2=1438455&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicBlockPackedReader.java Fri Jan 25 11:21:35 2013
@@ -44,7 +44,7 @@ public final class MonotonicBlockPackedR
blockShift = Integer.numberOfTrailingZeros(blockSize);
blockMask = blockSize - 1;
final int numBlocks = (int) (valueCount / blockSize) + (valueCount % blockSize == 0 ? 0 : 1);
- if (numBlocks * blockSize < valueCount) {
+ if ((long) numBlocks * blockSize < valueCount) {
throw new IllegalArgumentException("valueCount is too large for this block size");
}
minValues = new long[numBlocks];
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1438455&r1=1438454&r2=1438455&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Fri Jan 25 11:21:35 2013
@@ -36,11 +36,10 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.packed.PackedInts.Reader;
-
import org.junit.Ignore;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
@@ -1020,4 +1019,47 @@ public class TestPackedInts extends Luce
}
}
+ @Nightly
+ public void testBlockReaderOverflow() throws IOException {
+ final long valueCount = _TestUtil.nextLong(random(), 1L + Integer.MAX_VALUE, (long) Integer.MAX_VALUE * 2);
+ final int blockSize = 1 << _TestUtil.nextInt(random(), 20, 22);
+ final Directory dir = newDirectory();
+ final IndexOutput out = dir.createOutput("out.bin", IOContext.DEFAULT);
+ final BlockPackedWriter writer = new BlockPackedWriter(out, blockSize);
+ long value = random().nextInt() & 0xFFFFFFFFL;
+ long valueOffset = _TestUtil.nextLong(random(), 0, valueCount - 1);
+ for (long i = 0; i < valueCount; ) {
+ assertEquals(i, writer.ord());
+ if ((i & (blockSize - 1)) == 0 && (i + blockSize < valueOffset || i > valueOffset && i + blockSize < valueCount)) {
+ writer.addBlockOfZeros();
+ i += blockSize;
+ } else if (i == valueOffset) {
+ writer.add(value);
+ ++i;
+ } else {
+ writer.add(0);
+ ++i;
+ }
+ }
+ writer.finish();
+ out.close();
+ final IndexInput in = dir.openInput("out.bin", IOContext.DEFAULT);
+ final BlockPackedReaderIterator it = new BlockPackedReaderIterator(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
+ it.skip(valueOffset);
+ assertEquals(value, it.next());
+ in.seek(0L);
+ final BlockPackedReader reader = new BlockPackedReader(in, PackedInts.VERSION_CURRENT, blockSize, valueCount, random().nextBoolean());
+ assertEquals(value, reader.get(valueOffset));
+ for (int i = 0; i < 5; ++i) {
+ final long offset = _TestUtil.nextLong(random(), 0, valueCount - 1);
+ if (offset == valueOffset) {
+ assertEquals(value, reader.get(offset));
+ } else {
+ assertEquals(0, reader.get(offset));
+ }
+ }
+ in.close();
+ dir.close();
+ }
+
}