You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2018/02/07 15:39:30 UTC
nifi git commit: NIFI-4844: Adjust BigDecimal scale to the target
Avro schema
Repository: nifi
Updated Branches:
refs/heads/master dbbf78f22 -> 2b062e211
NIFI-4844: Adjust BigDecimal scale to the target Avro schema
- Applied the same scale adjustment not only to BigDecimal inputs, but
also to Double values.
Signed-off-by: Matthew Burgess <ma...@apache.org>
This closes #2450
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2b062e21
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2b062e21
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2b062e21
Branch: refs/heads/master
Commit: 2b062e211f36a0833c273e1a552d42723fdf54fe
Parents: dbbf78f
Author: Koji Kawamura <ij...@apache.org>
Authored: Tue Feb 6 13:52:48 2018 +0900
Committer: Matthew Burgess <ma...@apache.org>
Committed: Wed Feb 7 10:38:45 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 17 ++++----
.../org/apache/nifi/avro/TestAvroTypeUtil.java | 44 ++++++++++++++++++++
2 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/2b062e21/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index 48f661f..f7b8d9b 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -19,7 +19,6 @@ package org.apache.nifi.avro;
import java.io.IOException;
import java.math.BigDecimal;
-import java.math.MathContext;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
@@ -472,7 +471,7 @@ public class AvroTypeUtil {
/**
* Convert a raw value to an Avro object to serialize in Avro type system.
- * The counter-part method which reads an Avro object back to a raw value is {@link #normalizeValue(Object, Schema)}.
+ * The counter-part method which reads an Avro object back to a raw value is {@link #normalizeValue(Object, Schema, String)}.
*/
public static Object convertToAvroObject(final Object rawValue, final Schema fieldSchema) {
return convertToAvroObject(rawValue, fieldSchema, fieldSchema.getName());
@@ -531,18 +530,18 @@ public class AvroTypeUtil {
final LogicalType logicalType = fieldSchema.getLogicalType();
if (logicalType != null && LOGICAL_TYPE_DECIMAL.equals(logicalType.getName())) {
final LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) logicalType;
- final BigDecimal decimal;
+ final BigDecimal rawDecimal;
if (rawValue instanceof BigDecimal) {
- final BigDecimal rawDecimal = (BigDecimal) rawValue;
- final int desiredScale = decimalType.getScale();
- // If the desired scale is different than this value's coerce scale.
- decimal = rawDecimal.scale() == desiredScale ? rawDecimal : rawDecimal.setScale(desiredScale, BigDecimal.ROUND_HALF_UP);
+ rawDecimal = (BigDecimal) rawValue;
} else if (rawValue instanceof Double) {
- // Scale is adjusted based on precision. If double was 123.456 and precision is 5, then decimal would be 123.46.
- decimal = new BigDecimal((Double) rawValue, new MathContext(decimalType.getPrecision()));
+ rawDecimal = BigDecimal.valueOf((Double) rawValue);
} else {
throw new IllegalTypeConversionException("Cannot convert value " + rawValue + " of type " + rawValue.getClass() + " to a logical decimal");
}
+ // If the desired scale is different than this value's coerce scale.
+ final int desiredScale = decimalType.getScale();
+ final BigDecimal decimal = rawDecimal.scale() == desiredScale
+ ? rawDecimal : rawDecimal.setScale(desiredScale, BigDecimal.ROUND_HALF_UP);
return new Conversions.DecimalConversion().toBytes(decimal, fieldSchema, logicalType);
}
if (rawValue instanceof byte[]) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/2b062e21/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
index cf096aa..d644f7c 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
@@ -21,11 +21,17 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
+import org.apache.avro.Conversions;
+import org.apache.avro.LogicalTypes;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
@@ -259,4 +265,42 @@ public class TestAvroTypeUtil {
}
}
+ @Test
+ public void testToDecimalConversion() {
+ final LogicalTypes.Decimal decimalType = LogicalTypes.decimal(18, 8);
+ final Schema fieldSchema = Schema.create(Type.BYTES);
+ decimalType.addToSchema(fieldSchema);
+
+ final Map<Object, String> expects = new HashMap<>();
+
+ // Double to Decimal
+ expects.put(123d, "123.00000000");
+ // Double can not represent exact 1234567890.12345678, so use 1 less digit to test here.
+ expects.put(1234567890.12345678d, "1234567890.12345670");
+ expects.put(123456789.12345678d, "123456789.12345678");
+ expects.put(1234567890123456d, "1234567890123456.00000000");
+ // ROUND HALF UP.
+ expects.put(0.1234567890123456d, "0.12345679");
+
+
+ // BigDecimal to BigDecimal
+ expects.put(new BigDecimal("123"), "123.00000000");
+ expects.put(new BigDecimal("1234567890.12345678"), "1234567890.12345678");
+ expects.put(new BigDecimal("123456789012345678"), "123456789012345678.00000000");
+ // ROUND HALF UP.
+ expects.put(new BigDecimal("0.123456789012345678"), "0.12345679");
+
+
+ expects.forEach((rawValue, expect) -> {
+ final Object convertedValue = AvroTypeUtil.convertToAvroObject(rawValue, fieldSchema);
+
+ assertTrue(convertedValue instanceof ByteBuffer);
+ final ByteBuffer serializedBytes = (ByteBuffer) convertedValue;
+
+ final BigDecimal bigDecimal = new Conversions.DecimalConversion().fromBytes(serializedBytes, fieldSchema, decimalType);
+ assertEquals(String.format("%s should be converted to %s", rawValue, expect), expect, bigDecimal.toString());
+ });
+
+ }
+
}