You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by pr...@apache.org on 2020/11/25 06:36:48 UTC
[arrow] branch master updated: ARROW-10690: [Java] Fix
ComplexCopier bug for list vector
This is an automated email from the ASF dual-hosted git repository.
praveenbingo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 2b68a7b ARROW-10690: [Java] Fix ComplexCopier bug for list vector
2b68a7b is described below
commit 2b68a7b28c986cbf0328ae1258ec20e864315c77
Author: Projjal Chanda <ia...@pchanda.com>
AuthorDate: Wed Nov 25 12:05:38 2020 +0530
ARROW-10690: [Java] Fix ComplexCopier bug for list vector
When copying a list vector using ComplexCopier, if the target list vector is non-empty, the result vector is incorrect. For example, if src = [[1,2], [3, 4]], tgt = [[5, 6], [7, 8]], copying src to tgt gives tgt = [[5, 6, 1, 2], [3, 4]] instead of [[1,2], [3, 4]]
Similary when using copyFrom(/Safe) method if the vector at the index is non-empty, the listElement is appended instead. For above src and tgt vector, tgt.copyFrom(0, 0, src) gives [[5, 6, 1, 2], []] instead of [[1,2], [7, 8]]
Closes #8742 from projjal/listcomplexcopier and squashes the following commits:
15313199f <Projjal Chanda> move condition from list writer writer to list vector
71112eb48 <Projjal Chanda> fix complex copier
Authored-by: Projjal Chanda <ia...@pchanda.com>
Signed-off-by: Praveen <pr...@dremio.com>
---
.../apache/arrow/vector/complex/ListVector.java | 3 ++
.../vector/complex/impl/TestComplexCopier.java | 51 ++++++++++++++++++++++
2 files changed, 54 insertions(+)
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 1d674ca..a33acd2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -804,6 +804,9 @@ public class ListVector extends BaseRepeatedValueVector implements PromotableVec
while (index >= getValidityAndOffsetValueCapacity()) {
reallocValidityAndOffsetBuffers();
}
+ if (lastSet >= index) {
+ lastSet = index - 1;
+ }
for (int i = lastSet + 1; i <= index; i++) {
final int currentOffset = offsetBuffer.getInt(i * OFFSET_WIDTH);
offsetBuffer.setInt((i + 1) * OFFSET_WIDTH, currentOffset);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java
index 0c560ab..f314a98 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestComplexCopier.java
@@ -209,6 +209,57 @@ public class TestComplexCopier {
}
@Test
+ public void testCopyListVectorToANonEmptyList() {
+ try (ListVector from = ListVector.empty("v", allocator);
+ ListVector to = ListVector.empty("v", allocator)) {
+
+ UnionListWriter listWriter = from.getWriter();
+ listWriter.allocate();
+
+ for (int i = 0; i < COUNT; i++) {
+ listWriter.setPosition(i);
+ listWriter.startList();
+ listWriter.integer().writeInt(i);
+ listWriter.integer().writeInt(i * 2);
+ listWriter.endList();
+ }
+ from.setValueCount(COUNT);
+
+ // copy values
+ FieldReader in = from.getReader();
+ FieldWriter out = to.getWriter();
+ for (int i = 0; i < COUNT; i++) {
+ in.setPosition(i);
+ out.setPosition(i);
+ ComplexCopier.copy(in, out);
+ }
+ to.setValueCount(COUNT);
+ // validate equals
+ assertTrue(VectorEqualsVisitor.vectorEquals(from, to));
+
+ // Copy again to the target vector which is non-empty
+ for (int i = 0; i < COUNT; i++) {
+ in.setPosition(i);
+ out.setPosition(i);
+ ComplexCopier.copy(in, out);
+ }
+ to.setValueCount(COUNT);
+
+ // validate equals
+ assertTrue(VectorEqualsVisitor.vectorEquals(from, to));
+
+ // copy using copyFromSafe method
+ for (int i = 0; i < COUNT; i++) {
+ to.copyFromSafe(i, i, from);
+ }
+ to.setValueCount(COUNT);
+
+ // validate equals
+ assertTrue(VectorEqualsVisitor.vectorEquals(from, to));
+ }
+ }
+
+ @Test
public void testCopyListVectorWithNulls() {
try (ListVector from = ListVector.empty("v", allocator);
ListVector to = ListVector.empty("v", allocator)) {