You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2022/02/18 08:22:09 UTC
[arrow] branch master updated: ARROW-15501: [Java] Support validating decimal vectors
This is an automated email from the ASF dual-hosted git repository.
liyafan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 2a19660 ARROW-15501: [Java] Support validating decimal vectors
2a19660 is described below
commit 2a196601d520e7844f5cc70aa95f93a6fcb2d3bb
Author: liyafan82 <fa...@foxmail.com>
AuthorDate: Fri Feb 18 16:20:14 2022 +0800
ARROW-15501: [Java] Support validating decimal vectors
Support validating decimal vectors and check precisions.
Closes #12294 from liyafan82/fly_0129_dec
Authored-by: liyafan82 <fa...@foxmail.com>
Signed-off-by: liyafan82 <fa...@foxmail.com>
---
.../org/apache/arrow/vector/Decimal256Vector.java | 1 +
.../org/apache/arrow/vector/DecimalVector.java | 1 +
.../vector/validate/ValidateVectorTypeVisitor.java | 27 ++++++++++++++++++----
.../validate/TestValidateVectorTypeVisitor.java | 25 ++++++++++++++++++++
4 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java
index c5fef82..4341e41 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/Decimal256Vector.java
@@ -43,6 +43,7 @@ import io.netty.util.internal.PlatformDependent;
* maintained to track which elements in the vector are null.
*/
public final class Decimal256Vector extends BaseFixedWidthVector {
+ public static final int MAX_PRECISION = 76;
public static final byte TYPE_WIDTH = 32;
private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
private final FieldReader reader;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
index f988f4f..3768c65 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/DecimalVector.java
@@ -43,6 +43,7 @@ import io.netty.util.internal.PlatformDependent;
* maintained to track which elements in the vector are null.
*/
public final class DecimalVector extends BaseFixedWidthVector {
+ public static final int MAX_PRECISION = 38;
public static final byte TYPE_WIDTH = 16;
private static final boolean LITTLE_ENDIAN = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
private final FieldReader reader;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
index 65795b4..08e3ccc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java
@@ -116,6 +116,28 @@ public class ValidateVectorTypeVisitor implements VectorVisitor<Void, Void> {
"Expecting date unit %s, actual date unit %s.", expectedDateUnit, dateType.getUnit());
}
+ private void validateDecimalVector(ValueVector vector) {
+ validateOrThrow(vector.getField().getFieldType().getType() instanceof ArrowType.Decimal,
+ "Vector %s is not a decimal vector", vector.getClass());
+ ArrowType.Decimal decimalType = (ArrowType.Decimal) vector.getField().getFieldType().getType();
+ validateOrThrow(decimalType.getScale() >= 0, "The scale of decimal %s is negative.", decimalType.getScale());
+ validateOrThrow(decimalType.getScale() <= decimalType.getPrecision(),
+ "The scale of decimal %s is greater than the precision %s.",
+ decimalType.getScale(), decimalType.getPrecision());
+ switch (decimalType.getBitWidth()) {
+ case DecimalVector.TYPE_WIDTH * 8:
+ validateOrThrow(decimalType.getPrecision() >= 1 && decimalType.getPrecision() <= DecimalVector.MAX_PRECISION,
+ "Invalid precision %s for decimal 128.", decimalType.getPrecision());
+ break;
+ case Decimal256Vector.TYPE_WIDTH * 8:
+ validateOrThrow(decimalType.getPrecision() >= 1 && decimalType.getPrecision() <= Decimal256Vector.MAX_PRECISION,
+ "Invalid precision %s for decimal 256.", decimalType.getPrecision());
+ break;
+ default:
+ throw new ValidateUtil.ValidateException("Only decimal 128 or decimal 256 are supported for decimal types");
+ }
+ }
+
private void validateTimeVector(ValueVector vector, TimeUnit expectedTimeUnit, int expectedBitWidth) {
validateOrThrow(vector.getField().getFieldType().getType() instanceof ArrowType.Time,
"Vector %s is not a time vector.", vector.getClass());
@@ -188,10 +210,7 @@ public class ValidateVectorTypeVisitor implements VectorVisitor<Void, Void> {
validateVectorCommon(vector, ArrowType.Bool.class);
} else if (vector instanceof DecimalVector || vector instanceof Decimal256Vector) {
validateVectorCommon(vector, ArrowType.Decimal.class);
- ArrowType.Decimal arrowType = (ArrowType.Decimal) vector.getField().getType();
- validateOrThrow(arrowType.getScale() > 0, "The scale of decimal %s is not positive.", arrowType.getScale());
- validateOrThrow(arrowType.getPrecision() > 0, "The precision of decimal %S is not positive.",
- arrowType.getPrecision());
+ validateDecimalVector(vector);
} else if (vector instanceof DateDayVector) {
validateVectorCommon(vector, ArrowType.Date.class);
validateDateVector(vector, DateUnit.DAY);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java
index 7a0f12f..0ddd790 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorTypeVisitor.java
@@ -27,6 +27,7 @@ import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.DateDayVector;
import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.Decimal256Vector;
import org.apache.arrow.vector.DecimalVector;
import org.apache.arrow.vector.DurationVector;
import org.apache.arrow.vector.FixedSizeBinaryVector;
@@ -228,6 +229,30 @@ public class TestValidateVectorTypeVisitor {
}
@Test
+ public void testDecimalVector() {
+ testPositiveCase(() ->
+ new DecimalVector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(10, 10, 128)), allocator));
+ testPositiveCase(() ->
+ new DecimalVector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(38, 10, 128)), allocator));
+ testPositiveCase(() ->
+ new Decimal256Vector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(50, 10, 256)), allocator));
+ testPositiveCase(() ->
+ new Decimal256Vector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(76, 10, 256)), allocator));
+ testNegativeCase(() ->
+ new DecimalVector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(50, 10, 128)), allocator));
+ testNegativeCase(() ->
+ new Decimal256Vector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(100, 10, 256)), allocator));
+ testNegativeCase(() ->
+ new DecimalVector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(0, 10, 128)), allocator));
+ testNegativeCase(() ->
+ new Decimal256Vector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(-1, 10, 256)), allocator));
+ testNegativeCase(() ->
+ new Decimal256Vector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(30, 10, 64)), allocator));
+ testNegativeCase(() ->
+ new Decimal256Vector("dec", FieldType.nullable(ArrowType.Decimal.createDecimal(10, 20, 256)), allocator));
+ }
+
+ @Test
public void testVariableWidthVectorsPositive() {
testPositiveCase(() -> new VarCharVector("vector", allocator));
testPositiveCase(() -> new VarBinaryVector("vector", allocator));