You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2017/05/17 13:45:35 UTC
nifi git commit: NIFI-3920: Remove unnecessary code from AvroTypeUtil
Repository: nifi
Updated Branches:
refs/heads/master c50d51607 -> 33dc3e36f
NIFI-3920: Remove unnecessary code from AvroTypeUtil
- Removed remaining duplicate lines of code left by NIFI-3861 refactoring.
- Added test case that writes Avro record having union field.
This closes #1813.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/33dc3e36
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/33dc3e36
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/33dc3e36
Branch: refs/heads/master
Commit: 33dc3e36fbb7910b9d1ee05091b37b08f32aae6a
Parents: c50d516
Author: Koji Kawamura <ij...@apache.org>
Authored: Wed May 17 11:23:54 2017 +0900
Committer: Mark Payne <ma...@hotmail.com>
Committed: Wed May 17 09:45:21 2017 -0400
----------------------------------------------------------------------
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 18 ------------------
.../org/apache/nifi/avro/TestWriteAvroResult.java | 18 +++++++++++++++++-
2 files changed, 17 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/33dc3e36/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index 0682b34..6494fe5 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -386,24 +386,6 @@ public class AvroTypeUtil {
}
return avroRecord;
case UNION:
- // Ignore null types in union
- final List<Schema> nonNullFieldSchemas = getNonNullSubSchemas(fieldSchema);
-
- // If at least one non-null type exists, find the first compatible type
- if (nonNullFieldSchemas.size() >= 1) {
- for (final Schema nonNullFieldSchema : nonNullFieldSchemas) {
- final Object avroObject = convertToAvroObject(rawValue, nonNullFieldSchema, fieldName);
- final DataType desiredDataType = AvroTypeUtil.determineDataType(nonNullFieldSchema);
- if (DataTypeUtils.isCompatibleDataType(avroObject, desiredDataType)
- // For logical types those store with different type (e.g. BigDecimal as ByteBuffer), check compatibility using the original rawValue
- || (nonNullFieldSchema.getLogicalType() != null && DataTypeUtils.isCompatibleDataType(rawValue, desiredDataType))) {
- return avroObject;
- }
- }
-
- throw new IllegalTypeConversionException("Cannot convert value " + rawValue + " of type " + rawValue.getClass()
- + " because no compatible types exist in the UNION");
- }
return convertUnionFieldValue(rawValue, fieldSchema, schema -> convertToAvroObject(rawValue, schema, fieldName));
case ARRAY:
final Object[] objectArray = (Object[]) rawValue;
http://git-wip-us.apache.org/repos/asf/nifi/blob/33dc3e36/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java
index c9587f3..0a84aec 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java
@@ -39,9 +39,11 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.TimeZone;
import org.apache.avro.Conversions;
+import org.apache.avro.LogicalType;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData.Array;
import org.apache.avro.generic.GenericRecord;
@@ -68,6 +70,16 @@ public abstract class TestWriteAvroResult {
@Test
public void testLogicalTypes() throws IOException, ParseException {
final Schema schema = new Schema.Parser().parse(new File("src/test/resources/avro/logical-types.avsc"));
+ testLogicalTypes(schema);
+ }
+
+ @Test
+ public void testNullableLogicalTypes() throws IOException, ParseException {
+ final Schema schema = new Schema.Parser().parse(new File("src/test/resources/avro/logical-types-nullable.avsc"));
+ testLogicalTypes(schema);
+ }
+
+ private void testLogicalTypes(Schema schema) throws ParseException, IOException {
final WriteAvroResult writer = createWriter(schema);
final List<RecordField> fields = new ArrayList<>();
@@ -114,7 +126,11 @@ public abstract class TestWriteAvroResult {
assertEquals(17260, avroRecord.get("date"));
// Double value will be converted into logical decimal if Avro schema is defined as logical decimal.
final Schema decimalSchema = schema.getField("decimal").schema();
- final BigDecimal decimal = new Conversions.DecimalConversion().fromBytes((ByteBuffer) avroRecord.get("decimal"), decimalSchema, decimalSchema.getLogicalType());
+ final LogicalType logicalType = decimalSchema.getLogicalType() != null
+ ? decimalSchema.getLogicalType()
+ // Union type doesn't return logical type. Find the first logical type defined within the union.
+ : decimalSchema.getTypes().stream().map(s -> s.getLogicalType()).filter(Objects::nonNull).findFirst().get();
+ final BigDecimal decimal = new Conversions.DecimalConversion().fromBytes((ByteBuffer) avroRecord.get("decimal"), decimalSchema, logicalType);
assertEquals(expectedDecimal, decimal);
}
}