You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2018/12/28 11:11:36 UTC
[2/2] lucene-solr:branch_7x: LUCENE-8625: int overflow in
ByteBuffersDataInput.sliceBufferList
LUCENE-8625: int overflow in ByteBuffersDataInput.sliceBufferList
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a56297d7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a56297d7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a56297d7
Branch: refs/heads/branch_7x
Commit: a56297d712f73730df49f7d020c43e76ba394a0d
Parents: 4bcf34a
Author: Dawid Weiss <dw...@apache.org>
Authored: Fri Dec 28 10:54:16 2018 +0100
Committer: Dawid Weiss <dw...@apache.org>
Committed: Fri Dec 28 12:11:15 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../lucene/store/ByteBuffersDataInput.java | 10 ++--
.../lucene/store/TestByteBuffersDataInput.java | 49 ++++++++++++++++++++
3 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a56297d7/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2d4ca17..55aeb93ec 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -20,6 +20,9 @@ Bug fixes:
* LUCENE-8624: int overflow in ByteBuffersDataOutput.size(). (Mulugeta Mammo,
Dawid Weiss)
+* LUCENE-8625: int overflow in ByteBuffersDataInput.sliceBufferList. (Mulugeta Mammo,
+ Dawid Weiss)
+
New Features
* LUCENE-8026: ExitableDirectoryReader may now time out queries that run on
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a56297d7/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
index 9b4f4af..bc6cec5 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
@@ -290,17 +290,17 @@ public final class ByteBuffersDataInput extends DataInput implements Accountable
if (buffers.size() == 1) {
ByteBuffer cloned = buffers.get(0).asReadOnlyBuffer();
cloned.position(Math.toIntExact(cloned.position() + offset));
- cloned.limit(Math.toIntExact(length + cloned.position()));
+ cloned.limit(Math.toIntExact(cloned.position() + length));
return Arrays.asList(cloned);
} else {
long absStart = buffers.get(0).position() + offset;
- long absEnd = Math.toIntExact(absStart + length);
+ long absEnd = absStart + length;
int blockBytes = ByteBuffersDataInput.determineBlockPage(buffers);
int blockBits = Integer.numberOfTrailingZeros(blockBytes);
- int blockMask = (1 << blockBits) - 1;
+ long blockMask = (1L << blockBits) - 1;
- int endOffset = (int) absEnd & blockMask;
+ int endOffset = Math.toIntExact(absEnd & blockMask);
ArrayList<ByteBuffer> cloned =
buffers.subList(Math.toIntExact(absStart / blockBytes),
@@ -313,7 +313,7 @@ public final class ByteBuffersDataInput extends DataInput implements Accountable
cloned.add(ByteBuffer.allocate(0));
}
- cloned.get(0).position((int) absStart & blockMask);
+ cloned.get(0).position(Math.toIntExact(absStart & blockMask));
cloned.get(cloned.size() - 1).limit(endOffset);
return cloned;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a56297d7/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
index 5d3d7f6..20e3bd2 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestByteBuffersDataInput.java
@@ -21,6 +21,7 @@ import static org.junit.Assert.*;
import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.ArrayUtil;
@@ -203,4 +204,52 @@ public final class TestByteBuffersDataInput extends RandomizedTest {
in.readBytes(ByteBuffer.allocate(100), 100);
});
}
+
+ // https://issues.apache.org/jira/browse/LUCENE-8625
+ @Test
+ public void testSlicingLargeBuffers() throws IOException {
+ // Simulate a "large" (> 4GB) input by duplicating
+ // buffers with the same content.
+ int MB = 1024 * 1024;
+ byte [] pageBytes = randomBytesOfLength(4 * MB);
+ ByteBuffer page = ByteBuffer.wrap(pageBytes);
+
+ // Add some head shift on the first buffer.
+ final int shift = randomIntBetween(0, pageBytes.length / 2);
+
+ final long simulatedLength =
+ randomLongBetween(0, 2018) + 4L * Integer.MAX_VALUE;
+
+ List<ByteBuffer> buffers = new ArrayList<>();
+ long remaining = simulatedLength + shift;
+ while (remaining > 0) {
+ ByteBuffer bb = page.duplicate();
+ if (bb.remaining() > remaining) {
+ bb.limit(Math.toIntExact(bb.position() + remaining));
+ }
+ buffers.add(bb);
+ remaining -= bb.remaining();
+ }
+ buffers.get(0).position(shift);
+
+ ByteBuffersDataInput dst = new ByteBuffersDataInput(buffers);
+ assertEquals(simulatedLength, dst.size());
+
+ final long max = dst.size();
+ long offset = 0;
+ for (; offset < max; offset += randomIntBetween(MB, 4 * MB)) {
+ assertEquals(0, dst.slice(offset, 0).size());
+ assertEquals(1, dst.slice(offset, 1).size());
+
+ long window = Math.min(max - offset, 1024);
+ ByteBuffersDataInput slice = dst.slice(offset, window);
+ assertEquals(window, slice.size());
+
+ // Sanity check of the content against original pages.
+ for (int i = 0; i < window; i++) {
+ byte expected = pageBytes[(int) ((shift + offset + i) % pageBytes.length)];
+ assertEquals(expected, slice.readByte(i));
+ }
+ }
+ }
}