You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2018/01/11 20:05:16 UTC
nifi git commit: NIFI-4441: patch avro maps in union types. This
closes #2207.
Repository: nifi
Updated Branches:
refs/heads/master 39a484b63 -> 5f7bd81af
NIFI-4441: patch avro maps in union types. This closes #2207.
Signed-off-by: Mark Payne <ma...@hotmail.com>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/5f7bd81a
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/5f7bd81a
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/5f7bd81a
Branch: refs/heads/master
Commit: 5f7bd81af97523b6b25f206cd0810c148d8dcd4a
Parents: 39a484b
Author: Patrice Freydiere <fr...@gmail.com>
Authored: Wed Oct 11 22:17:15 2017 +0200
Committer: Mark Payne <ma...@hotmail.com>
Committed: Thu Jan 11 15:04:42 2018 -0500
----------------------------------------------------------------------
.../nifi-avro-record-utils/pom.xml | 14 +++++++++++++
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 10 ++++++++++
.../org/apache/nifi/avro/TestAvroTypeUtil.java | 20 +++++++++++++++++++
.../resources/org/apache/nifi/avro/data.avro | Bin 0 -> 269 bytes
.../resources/org/apache/nifi/avro/schema.json | 20 +++++++++++++++++++
5 files changed, 64 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/5f7bd81a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml
index c8fd38d..cec2d9d 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/pom.xml
@@ -42,4 +42,18 @@
<artifactId>nifi-record</artifactId>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes combine.children="append">
+ <exclude>src/test/resources/org/apache/nifi/avro/data.avro</exclude>
+ <exclude>src/test/resources/org/apache/nifi/avro/schema.json</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/5f7bd81a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index cc1fd38..adbb6e3 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -626,6 +626,7 @@ public class AvroTypeUtil {
final Map<String, Object> values = new HashMap<>(recordSchema.getFieldCount());
for (final RecordField recordField : recordSchema.getFields()) {
+
Object value = avroRecord.get(recordField.getFieldName());
if (value == null) {
for (final String alias : recordField.getAliases()) {
@@ -637,6 +638,7 @@ public class AvroTypeUtil {
}
final String fieldName = recordField.getFieldName();
+ try {
final Field avroField = avroRecord.getSchema().getField(fieldName);
if (avroField == null) {
values.put(fieldName, null);
@@ -650,6 +652,10 @@ public class AvroTypeUtil {
final Object coercedValue = DataTypeUtils.convertType(rawValue, desiredType, fieldName);
values.put(fieldName, coercedValue);
+ } catch (Exception ex) {
+ logger.debug("fail to convert field " + fieldName, ex );
+ throw ex;
+ }
}
return values;
@@ -716,6 +722,10 @@ public class AvroTypeUtil {
return true;
}
break;
+ case MAP:
+ if (value instanceof Map) {
+ return true;
+ }
}
return DataTypeUtils.isCompatibleDataType(value, dataType);
http://git-wip-us.apache.org/repos/asf/nifi/blob/5f7bd81a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
index b017829..cf096aa 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
@@ -20,6 +20,7 @@ package org.apache.nifi.avro;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -28,6 +29,9 @@ import java.util.Optional;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileStream;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
import org.apache.nifi.schema.access.SchemaNotFoundException;
import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.record.DataType;
@@ -239,4 +243,20 @@ public class TestAvroTypeUtil {
Assert.assertEquals(recordASchema, ((RecordDataType)recordBParentField.get().getDataType()).getChildSchema());
}
+ @Test
+ public void testMapWithNullSchema() throws IOException {
+
+ Schema recursiveSchema = new Schema.Parser().parse(getClass().getResourceAsStream("schema.json"));
+
+ // Make sure the following doesn't throw an exception
+ RecordSchema recordASchema = AvroTypeUtil.createSchema(recursiveSchema.getTypes().get(0));
+
+ // check the fix with the proper file
+ try(DataFileStream<GenericRecord> r = new DataFileStream<>(getClass().getResourceAsStream("data.avro"),
+ new GenericDatumReader<>())) {
+ GenericRecord n= r.next();
+ AvroTypeUtil.convertAvroRecordToMap(n, recordASchema);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/5f7bd81a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/data.avro
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/data.avro b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/data.avro
new file mode 100644
index 0000000..139f244
Binary files /dev/null and b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/data.avro differ
http://git-wip-us.apache.org/repos/asf/nifi/blob/5f7bd81a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/schema.json
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/schema.json b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/schema.json
new file mode 100644
index 0000000..f6fd74b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/resources/org/apache/nifi/avro/schema.json
@@ -0,0 +1,20 @@
+[ {
+ "namespace" : "net.a",
+ "type" : "record",
+ "name" : "O",
+ "fields" : [ {
+ "name" : "hash",
+ "type" : [ "null", {
+ "type" : "map",
+ "values" : "string"
+ } ]
+ } ]
+}, {
+ "namespace" : "net.a",
+ "type" : "record",
+ "name" : "A",
+ "fields" : [ {
+ "name" : "o",
+ "type" : [ "null", "O" ]
+ }]
+} ]
\ No newline at end of file