You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ks...@apache.org on 2020/10/19 18:01:08 UTC

[arrow] 13/30: ARROW-10294: [Java] Resolve problems of DecimalVector APIs on ArrowBufs

This is an automated email from the ASF dual-hosted git repository.

kszucs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git

commit 22027c7b141fa2fbe168c2d31b5aa236caf62bd7
Author: liyafan82 <fa...@foxmail.com>
AuthorDate: Thu Oct 15 22:17:38 2020 -0700

    ARROW-10294: [Java] Resolve problems of DecimalVector APIs on ArrowBufs
    
    Unlike other fixed width vectors, DecimalVectors have some APIs that directly manipulate an ArrowBuf (e.g. `void set(int index, int isSet, int start, ArrowBuf buffer)`.
    
    After supporting 64-bit ArrowBufs, we need to adjust such APIs so that they work properly.
    
    Closes #8455 from liyafan82/fly_1012_dec
    
    Authored-by: liyafan82 <fa...@foxmail.com>
    Signed-off-by: Micah Kornfield <em...@gmail.com>
---
 .../src/main/codegen/data/ValueVectorTypes.tdd      |  2 +-
 .../src/main/codegen/templates/ComplexWriters.java  |  4 ++--
 .../codegen/templates/UnionFixedSizeListWriter.java |  2 +-
 .../src/main/codegen/templates/UnionListWriter.java |  4 ++--
 .../java/org/apache/arrow/vector/DecimalVector.java | 12 ++++++------
 .../arrow/vector/complex/impl/PromotableWriter.java |  2 +-
 .../apache/arrow/vector/util/DecimalUtility.java    |  2 +-
 .../org/apache/arrow/vector/ITTestLargeVector.java  | 21 ++++++++++++++++++++-
 8 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index b9e0529..7612d36 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -125,7 +125,7 @@
           maxPrecisionDigits: 38, nDecimalDigits: 4, friendlyType: "BigDecimal",
           typeParams: [ {name: "scale", type: "int"}, { name: "precision", type: "int"}],
           arrowType: "org.apache.arrow.vector.types.pojo.ArrowType.Decimal",
-          fields: [{name: "start", type: "int"}, {name: "buffer", type: "ArrowBuf"}]
+          fields: [{name: "start", type: "long"}, {name: "buffer", type: "ArrowBuf"}]
         }
       ]
     },
diff --git a/java/vector/src/main/codegen/templates/ComplexWriters.java b/java/vector/src/main/codegen/templates/ComplexWriters.java
index ab99ac3..5f5025f 100644
--- a/java/vector/src/main/codegen/templates/ComplexWriters.java
+++ b/java/vector/src/main/codegen/templates/ComplexWriters.java
@@ -139,12 +139,12 @@ public class ${eName}WriterImpl extends AbstractFieldWriter {
     vector.setValueCount(idx() + 1);
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer){
+  public void writeDecimal(long start, ArrowBuf buffer){
     vector.setSafe(idx(), 1, start, buffer);
     vector.setValueCount(idx() + 1);
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType){
+  public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType){
     DecimalUtility.checkPrecisionAndScale(((ArrowType.Decimal) arrowType).getPrecision(),
       ((ArrowType.Decimal) arrowType).getScale(), vector.getPrecision(), vector.getScale());
     vector.setSafe(idx(), 1, start, buffer);
diff --git a/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java b/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
index 0574dcf..94c7d8f 100644
--- a/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionFixedSizeListWriter.java
@@ -189,7 +189,7 @@ public class UnionFixedSizeListWriter extends AbstractFieldWriter {
     writer.writeNull();
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
+  public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) {
     if (writer.idx() >= (idx() + 1) * listSize) {
       throw new IllegalStateException(String.format("values at index %s is greater than listSize %s", idx(), listSize));
     }
diff --git a/java/vector/src/main/codegen/templates/UnionListWriter.java b/java/vector/src/main/codegen/templates/UnionListWriter.java
index a266443..bb0cff4 100644
--- a/java/vector/src/main/codegen/templates/UnionListWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionListWriter.java
@@ -204,12 +204,12 @@ public class Union${listName}Writer extends AbstractFieldWriter {
     writer.writeNull();
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
+  public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) {
     writer.writeDecimal(start, buffer, arrowType);
     writer.setPosition(writer.idx()+1);
   }
 
-  public void writeDecimal(int start, ArrowBuf buffer) {
+  public void writeDecimal(long start, ArrowBuf buffer) {
     writer.writeDecimal(start, buffer);
     writer.setPosition(writer.idx()+1);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index 554e174..04344c3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -246,7 +246,7 @@ public final class DecimalVector extends BaseFixedWidthVector {
    * @param start    start index of data in the buffer
    * @param buffer   ArrowBuf containing decimal value.
    */
-  public void set(int index, int start, ArrowBuf buffer) {
+  public void set(int index, long start, ArrowBuf buffer) {
     BitVectorHelper.setBit(validityBuffer, index);
     valueBuffer.setBytes((long) index * TYPE_WIDTH, buffer, start, TYPE_WIDTH);
   }
@@ -258,7 +258,7 @@ public final class DecimalVector extends BaseFixedWidthVector {
    * @param buffer contains the decimal in little endian bytes
    * @param length length of the value in the buffer
    */
-  public void setSafe(int index, int start, ArrowBuf buffer, int length) {
+  public void setSafe(int index, long start, ArrowBuf buffer, int length) {
     handleSafe(index);
     BitVectorHelper.setBit(validityBuffer, index);
 
@@ -285,7 +285,7 @@ public final class DecimalVector extends BaseFixedWidthVector {
    * @param buffer contains the decimal in big endian bytes
    * @param length length of the value in the buffer
    */
-  public void setBigEndianSafe(int index, int start, ArrowBuf buffer, int length) {
+  public void setBigEndianSafe(int index, long start, ArrowBuf buffer, int length) {
     handleSafe(index);
     BitVectorHelper.setBit(validityBuffer, index);
 
@@ -394,7 +394,7 @@ public final class DecimalVector extends BaseFixedWidthVector {
    * @param start    start index of data in the buffer
    * @param buffer   ArrowBuf containing decimal value.
    */
-  public void setSafe(int index, int start, ArrowBuf buffer) {
+  public void setSafe(int index, long start, ArrowBuf buffer) {
     handleSafe(index);
     set(index, start, buffer);
   }
@@ -460,7 +460,7 @@ public final class DecimalVector extends BaseFixedWidthVector {
    * @param start start position of the value in the buffer
    * @param buffer buffer containing the value to be stored in the vector
    */
-  public void set(int index, int isSet, int start, ArrowBuf buffer) {
+  public void set(int index, int isSet, long start, ArrowBuf buffer) {
     if (isSet > 0) {
       set(index, start, buffer);
     } else {
@@ -478,7 +478,7 @@ public final class DecimalVector extends BaseFixedWidthVector {
    * @param start start position of the value in the buffer
    * @param buffer buffer containing the value to be stored in the vector
    */
-  public void setSafe(int index, int isSet, int start, ArrowBuf buffer) {
+  public void setSafe(int index, int isSet, long start, ArrowBuf buffer) {
     handleSafe(index);
     set(index, isSet, start, buffer);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 6f40836..51decee 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -320,7 +320,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
   }
 
   @Override
-  public void writeDecimal(int start, ArrowBuf buffer, ArrowType arrowType) {
+  public void writeDecimal(long start, ArrowBuf buffer, ArrowType arrowType) {
     getWriter(MinorType.DECIMAL, new ArrowType.Decimal(MAX_DECIMAL_PRECISION,
         ((ArrowType.Decimal) arrowType).getScale())).writeDecimal(start, buffer, arrowType);
   }
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
index 711fa3b..36c988f 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/DecimalUtility.java
@@ -42,7 +42,7 @@ public class DecimalUtility {
   public static BigDecimal getBigDecimalFromArrowBuf(ArrowBuf bytebuf, int index, int scale) {
     byte[] value = new byte[DECIMAL_BYTE_LENGTH];
     byte temp;
-    final int startIndex = index * DECIMAL_BYTE_LENGTH;
+    final long startIndex = (long) index * DECIMAL_BYTE_LENGTH;
 
     // Decimal stored as little endian, need to swap bytes to make BigDecimal
     bytebuf.getBytes(startIndex, value, 0, DECIMAL_BYTE_LENGTH);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
index 8b824d6..19648dc 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ITTestLargeVector.java
@@ -21,9 +21,12 @@ import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.math.BigDecimal;
+
 import org.apache.arrow.memory.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.holders.NullableDecimalHolder;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -114,7 +117,7 @@ public class ITTestLargeVector {
     final int vecLength = (int) (bufSize / DecimalVector.TYPE_WIDTH);
 
     try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
-         DecimalVector largeVec = new DecimalVector("vec", allocator, 38, 16)) {
+         DecimalVector largeVec = new DecimalVector("vec", allocator, 38, 0)) {
       largeVec.allocateNew(vecLength);
 
       logger.trace("Successfully allocated a vector with capacity {}", vecLength);
@@ -139,6 +142,22 @@ public class ITTestLargeVector {
         }
       }
       logger.trace("Successfully read {} values", vecLength);
+
+      // try setting values with a large offset in the buffer
+      largeVec.set(vecLength - 1, 12345L);
+      assertEquals(12345L, largeVec.getObject(vecLength - 1).longValue());
+
+      NullableDecimalHolder holder = new NullableDecimalHolder();
+      holder.buffer = largeVec.valueBuffer;
+      holder.isSet = 1;
+      holder.start = (long) (vecLength - 1) * largeVec.getTypeWidth();
+      assertTrue(holder.start > Integer.MAX_VALUE);
+      largeVec.set(0, holder);
+
+      BigDecimal decimal = largeVec.getObject(0);
+      assertEquals(12345L, decimal.longValue());
+
+      logger.trace("Successfully setting values from large offsets");
     }
     logger.trace("Successfully released the large vector.");
   }