You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ra...@apache.org on 2019/08/12 08:54:49 UTC
[arrow] branch master updated: ARROW-6200: [Java] Method
getBufferSizeFor in BaseRepeatedValueVector/ListVector not correct
This is an automated email from the ASF dual-hosted git repository.
ravindra pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 6381043 ARROW-6200: [Java] Method getBufferSizeFor in BaseRepeatedValueVector/ListVector not correct
6381043 is described below
commit 6381043a6ea66ac2f8c69cdaabf0b5c96ee66152
Author: tianchen <ni...@alibaba-inc.com>
AuthorDate: Mon Aug 12 14:24:21 2019 +0530
ARROW-6200: [Java] Method getBufferSizeFor in BaseRepeatedValueVector/ListVector not correct
Related to ARROW-6200.
>Currently, getBufferSizeFor in BaseRepeatedValueVector implemented as below:
if (valueCount == 0) {
return 0;
}
return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSizeFor(valueCount);
Here vector.getBufferSizeFor(valueCount) seems not right which should be
>int innerVectorValueCount = offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
vector.getBufferSizeFor(innerVectorValueCount)
ListVector has the same problem.
Closes #5060 from tianchen92/ARROW-6200 and squashes the following commits:
f0bdbfb69 <tianchen> reuse code
90900e785 <tianchen> ARROW-6200: Method getBufferSizeFor in BaseRepeatedValueVector/ListVector not correct
Authored-by: tianchen <ni...@alibaba-inc.com>
Signed-off-by: Pindikura Ravindra <ra...@dremio.com>
---
.../vector/complex/BaseRepeatedValueVector.java | 6 ++--
.../apache/arrow/vector/complex/ListVector.java | 10 ++++++
.../org/apache/arrow/vector/TestListVector.java | 37 ++++++++++++++++++++++
3 files changed, 51 insertions(+), 2 deletions(-)
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index dbc7236..581f5d8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -209,7 +209,7 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
@Override
public int getBufferSize() {
- if (getValueCount() == 0) {
+ if (valueCount == 0) {
return 0;
}
return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSize();
@@ -221,7 +221,9 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
return 0;
}
- return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSizeFor(valueCount);
+ int innerVectorValueCount = offsetBuffer.getInt(valueCount * OFFSET_WIDTH);
+
+ return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSizeFor(innerVectorValueCount);
}
@Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index f07fbdb..d6935de 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -603,6 +603,16 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
}
@Override
+ public int getBufferSizeFor(int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+ final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
+
+ return super.getBufferSizeFor(valueCount) + validityBufferSize;
+ }
+
+ @Override
public Field getField() {
return new Field(getName(), fieldType, Collections.singletonList(getDataVector().getField()));
}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 96372c0..f80237b 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -26,6 +26,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.BaseRepeatedValueVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.complex.reader.FieldReader;
@@ -871,4 +872,40 @@ public class TestListVector {
assertEquals(new Long(8), resultSet.get(0));
}
}
+
+ @Test
+ public void testGetBufferSizeFor() {
+ try (final ListVector vector = ListVector.empty("list", allocator)) {
+
+ UnionListWriter writer = vector.getWriter();
+ writer.allocate();
+
+ //set some values
+ writeIntValues(writer, new int[] {1, 2});
+ writeIntValues(writer, new int[] {3, 4});
+ writeIntValues(writer, new int[] {5, 6});
+ writeIntValues(writer, new int[] {7, 8, 9, 10});
+ writeIntValues(writer, new int[] {11, 12, 13, 14});
+ writer.setValueCount(5);
+
+ IntVector dataVector = (IntVector) vector.getDataVector();
+ int[] indices = new int[] {0, 2, 4, 6, 10, 14};
+
+ for (int valueCount = 1; valueCount <= 5; valueCount++) {
+ int validityBufferSize = BitVectorHelper.getValidityBufferSize(valueCount);
+ int offsetBufferSize = (valueCount + 1) * BaseRepeatedValueVector.OFFSET_WIDTH;
+
+ int expectedSize = validityBufferSize + offsetBufferSize + dataVector.getBufferSizeFor(indices[valueCount]);
+ assertEquals(expectedSize, vector.getBufferSizeFor(valueCount));
+ }
+ }
+ }
+
+ private void writeIntValues(UnionListWriter writer, int[] values) {
+ writer.startList();
+ for (int v: values) {
+ writer.integer().writeInt(v);
+ }
+ writer.endList();
+ }
}