You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2020/12/10 06:15:45 UTC
[arrow] branch master updated: ARROW-10860: [Java] Avoid integer
overflow for generated classes in Vector
This is an automated email from the ASF dual-hosted git repository.
liyafan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b0b9269 ARROW-10860: [Java] Avoid integer overflow for generated classes in Vector
b0b9269 is described below
commit b0b926956626da72202c6520c0fd22184aa2b71e
Author: Kazuaki Ishizaki <is...@jp.ibm.com>
AuthorDate: Thu Dec 10 14:13:48 2020 +0800
ARROW-10860: [Java] Avoid integer overflow for generated classes in Vector
For the current implementation in the templates for Vector, `int * int` multiplication is used to calculate a buffer offset. The result may be larger than Integer.MAX_VALUE, which will lead to integer overflow and unexpected behaviors.
This PR is like a follow-up of #8721.
Closes #8876 from kiszk/ARROW-10860
Authored-by: Kazuaki Ishizaki <is...@jp.ibm.com>
Signed-off-by: liyafan82 <fa...@foxmail.com>
---
.../main/codegen/templates/DenseUnionVector.java | 26 +++++++++++-----------
.../main/codegen/templates/UnionListWriter.java | 8 +++----
2 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/java/vector/src/main/codegen/templates/DenseUnionVector.java b/java/vector/src/main/codegen/templates/DenseUnionVector.java
index de4d60d..cf56b51 100644
--- a/java/vector/src/main/codegen/templates/DenseUnionVector.java
+++ b/java/vector/src/main/codegen/templates/DenseUnionVector.java
@@ -210,7 +210,7 @@ public class DenseUnionVector implements FieldVector {
typeBuffer.writerIndex(valueCount * TYPE_WIDTH);
offsetBuffer.readerIndex(0);
- offsetBuffer.writerIndex(valueCount * OFFSET_WIDTH);
+ offsetBuffer.writerIndex((long) valueCount * OFFSET_WIDTH);
}
@Override
@@ -402,7 +402,7 @@ public class DenseUnionVector implements FieldVector {
}
public int getOffset(int index) {
- return offsetBuffer.getInt(index * OFFSET_WIDTH);
+ return offsetBuffer.getInt((long) index * OFFSET_WIDTH);
}
private void reallocTypeBuffer() {
@@ -546,9 +546,9 @@ public class DenseUnionVector implements FieldVector {
public void copyFrom(int inIndex, int outIndex, ValueVector from) {
Preconditions.checkArgument(this.getMinorType() == from.getMinorType());
DenseUnionVector fromCast = (DenseUnionVector) from;
- int inOffset = fromCast.offsetBuffer.getInt(inIndex * OFFSET_WIDTH);
+ int inOffset = fromCast.offsetBuffer.getInt((long) inIndex * OFFSET_WIDTH);
fromCast.getReader().setPosition(inOffset);
- int outOffset = offsetBuffer.getInt(outIndex * OFFSET_WIDTH);
+ int outOffset = offsetBuffer.getInt((long) outIndex * OFFSET_WIDTH);
getWriter().setPosition(outOffset);
ComplexCopier.copy(fromCast.reader, writer);
}
@@ -630,7 +630,7 @@ public class DenseUnionVector implements FieldVector {
to.typeBuffer = refManager.transferOwnership(slicedBuffer, to.allocator).getTransferredBuffer();
// transfer offset byffer
- while (to.offsetBuffer.capacity() < length * OFFSET_WIDTH) {
+ while (to.offsetBuffer.capacity() < (long) length * OFFSET_WIDTH) {
to.reallocOffsetBuffer();
}
@@ -643,10 +643,10 @@ public class DenseUnionVector implements FieldVector {
for (int i = startIndex; i < startIndex + length; i++) {
byte typeId = typeBuffer.getByte(i);
- to.offsetBuffer.setInt((i - startIndex) * OFFSET_WIDTH, typeCounts[typeId]);
+ to.offsetBuffer.setInt((long) (i - startIndex) * OFFSET_WIDTH, typeCounts[typeId]);
typeCounts[typeId] += 1;
if (typeStarts[typeId] == -1) {
- typeStarts[typeId] = offsetBuffer.getInt(i * OFFSET_WIDTH);
+ typeStarts[typeId] = offsetBuffer.getInt((long) i * OFFSET_WIDTH);
}
}
@@ -697,8 +697,8 @@ public class DenseUnionVector implements FieldVector {
if (count == 0) {
return 0;
}
- return count * TYPE_WIDTH + count * OFFSET_WIDTH + DataSizeRoundingUtil.divideBy8Ceil(count)
- + internalStruct.getBufferSizeFor(count);
+ return (int) (count * TYPE_WIDTH + (long) count * OFFSET_WIDTH
+ + DataSizeRoundingUtil.divideBy8Ceil(count) + internalStruct.getBufferSizeFor(count));
}
@Override
@@ -736,7 +736,7 @@ public class DenseUnionVector implements FieldVector {
public Object getObject(int index) {
ValueVector vector = getVector(index);
if (vector != null) {
- int offset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+ int offset = offsetBuffer.getInt((long) index * OFFSET_WIDTH);
return vector.isNull(offset) ? null : vector.getObject(offset);
}
return null;
@@ -799,7 +799,7 @@ public class DenseUnionVector implements FieldVector {
if (writer == null) {
writer = new DenseUnionWriter(DenseUnionVector.this);
}
- int offset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+ int offset = offsetBuffer.getInt((long) index * OFFSET_WIDTH);
MinorType type = reader.getMinorType();
writer.setPosition(offset);
byte typeId = holder.typeId;
@@ -843,7 +843,7 @@ public class DenseUnionVector implements FieldVector {
int offset = vector.getValueCount();
vector.setValueCount(offset + 1);
vector.setSafe(offset, holder);
- offsetBuffer.setInt(index * OFFSET_WIDTH, offset);
+ offsetBuffer.setInt((long) index * OFFSET_WIDTH, offset);
}
</#if>
</#list>
@@ -869,7 +869,7 @@ public class DenseUnionVector implements FieldVector {
if (isNull(index)) {
return 0;
}
- int offset = offsetBuffer.getInt(index * OFFSET_WIDTH);
+ int offset = offsetBuffer.getInt((long) index * OFFSET_WIDTH);
return getVector(index).hashCode(offset, hasher);
}
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index 0283732..15c6019 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -180,13 +180,13 @@ public class Union${listName}Writer extends AbstractFieldWriter {
@Override
public void startList() {
vector.startNewValue(idx());
- writer.setPosition(checkedCastToInt(vector.getOffsetBuffer().getLong(((long) idx() + 1L) * OFFSET_WIDTH)));
+ writer.setPosition(checkedCastToInt(vector.getOffsetBuffer().getLong((idx() + 1L) * OFFSET_WIDTH)));
listStarted = true;
}
@Override
public void endList() {
- vector.getOffsetBuffer().setLong(((long) idx() + 1L) * OFFSET_WIDTH, writer.idx());
+ vector.getOffsetBuffer().setLong((idx() + 1L) * OFFSET_WIDTH, writer.idx());
setPosition(idx() + 1);
listStarted = false;
}
@@ -194,13 +194,13 @@ public class Union${listName}Writer extends AbstractFieldWriter {
@Override
public void startList() {
vector.startNewValue(idx());
- writer.setPosition(vector.getOffsetBuffer().getInt((idx() + 1) * OFFSET_WIDTH));
+ writer.setPosition(vector.getOffsetBuffer().getInt((idx() + 1L) * OFFSET_WIDTH));
listStarted = true;
}
@Override
public void endList() {
- vector.getOffsetBuffer().setInt((idx() + 1) * OFFSET_WIDTH, writer.idx());
+ vector.getOffsetBuffer().setInt((idx() + 1L) * OFFSET_WIDTH, writer.idx());
setPosition(idx() + 1);
listStarted = false;
}