You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ma...@apache.org on 2017/05/12 00:25:34 UTC
nifi git commit: NIFI-3871: Convert Avro map values
Repository: nifi
Updated Branches:
refs/heads/master 0f693e944 -> 6e4db6b11
NIFI-3871: Convert Avro map values
This closes #1787.
- When converting from a raw value to an Avro object, convert the values
of any Avro map types so that they can be complex types like other
records.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/6e4db6b1
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/6e4db6b1
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/6e4db6b1
Branch: refs/heads/master
Commit: 6e4db6b11a9177052d193fcd66b01dad1fa645d2
Parents: 0f693e9
Author: Steve Champagne <ch...@gmail.com>
Authored: Thu May 11 16:39:38 2017 -0500
Committer: Mark Payne <ma...@hotmail.com>
Committed: Thu May 11 20:25:19 2017 -0400
----------------------------------------------------------------------
.../java/org/apache/nifi/avro/AvroTypeUtil.java | 8 +++++++-
.../org/apache/nifi/avro/TestWriteAvroResult.java | 17 ++++++++++++++++-
.../src/test/resources/avro/datatypes.avsc | 10 ++++++++--
3 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/6e4db6b1/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index 38e87a8..52d749f 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -327,7 +327,13 @@ public class AvroTypeUtil {
return map;
} else if (rawValue instanceof Map) {
- return rawValue;
+ final Map<String, Object> objectMap = (Map<String, Object>) rawValue;
+ final Map<String, Object> map = new HashMap<>(objectMap.size());
+ for (final String s : objectMap.keySet()) {
+ final Object converted = convertToAvroObject(objectMap.get(s), fieldSchema.getValueType(), fieldName);
+ map.put(s, converted);
+ }
+ return map;
} else {
throw new IllegalTypeConversionException("Cannot convert value " + rawValue + " of type " + rawValue.getClass() + " to a Map");
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/6e4db6b1/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java
index 409ede2..dc5d943 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/avro/TestWriteAvroResult.java
@@ -45,6 +45,7 @@ import org.apache.avro.generic.GenericData.Array;
import org.apache.avro.generic.GenericRecord;
import org.apache.nifi.serialization.SimpleRecordSchema;
import org.apache.nifi.serialization.WriteResult;
+import org.apache.nifi.serialization.record.DataType;
import org.apache.nifi.serialization.record.MapRecord;
import org.apache.nifi.serialization.record.Record;
import org.apache.nifi.serialization.record.RecordField;
@@ -115,6 +116,7 @@ public abstract class TestWriteAvroResult {
final List<RecordField> subRecordFields = Collections.singletonList(new RecordField("field1", RecordFieldType.STRING.getDataType()));
final RecordSchema subRecordSchema = new SimpleRecordSchema(subRecordFields);
+ final DataType subRecordDataType = RecordFieldType.RECORD.getRecordDataType(subRecordSchema);
final List<RecordField> fields = new ArrayList<>();
fields.add(new RecordField("string", RecordFieldType.STRING.getDataType()));
@@ -126,11 +128,15 @@ public abstract class TestWriteAvroResult {
fields.add(new RecordField("bytes", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.BYTE.getDataType())));
fields.add(new RecordField("nullOrLong", RecordFieldType.LONG.getDataType()));
fields.add(new RecordField("array", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.INT.getDataType())));
- fields.add(new RecordField("record", RecordFieldType.RECORD.getRecordDataType(subRecordSchema)));
+ fields.add(new RecordField("record", subRecordDataType));
+ fields.add(new RecordField("map", RecordFieldType.MAP.getMapDataType(subRecordDataType)));
final RecordSchema recordSchema = new SimpleRecordSchema(fields);
final Record innerRecord = new MapRecord(subRecordSchema, Collections.singletonMap("field1", "hello"));
+ final Map<String, Object> innerMap = new HashMap<>();
+ innerMap.put("key1", innerRecord);
+
final Map<String, Object> values = new HashMap<>();
values.put("string", "hello");
values.put("int", 8);
@@ -142,6 +148,7 @@ public abstract class TestWriteAvroResult {
values.put("nullOrLong", null);
values.put("array", new Integer[] {1, 2, 3});
values.put("record", innerRecord);
+ values.put("map", innerMap);
final Record record = new MapRecord(recordSchema, values);
@@ -188,6 +195,14 @@ public abstract class TestWriteAvroResult {
} else if (recordValue instanceof byte[]) {
final ByteBuffer bb = ByteBuffer.wrap((byte[]) recordValue);
assertEquals(fieldName + " not equal", bb, avroValue);
+ } else if (recordValue instanceof Map) {
+ assertTrue(fieldName + " should have been instanceof Map", avroValue instanceof Map);
+ final Map<?, ?> avroMap = (Map<?, ?>) avroValue;
+ final Map<?, ?> recordMap = (Map<?, ?>) recordValue;
+ assertEquals(fieldName + " not equal", recordMap.size(), avroMap.size());
+ for (Object s : avroMap.keySet()) {
+ assertMatch((Record) recordMap.get(s.toString()), (GenericRecord) avroMap.get(s));
+ }
} else if (recordValue instanceof Record) {
assertMatch((Record) recordValue, (GenericRecord) avroValue);
} else {
http://git-wip-us.apache.org/repos/asf/nifi/blob/6e4db6b1/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/avro/datatypes.avsc
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/avro/datatypes.avsc b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/avro/datatypes.avsc
index cc7f60e..4d26b64 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/avro/datatypes.avsc
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/avro/datatypes.avsc
@@ -42,6 +42,12 @@
{ "name": "field1", "type": "string" }
]
}
- }
+ }, {
+ "name": "map",
+ "type": {
+ "type": "map",
+ "values": "subRecord"
+ }
+ }
]
-}
\ No newline at end of file
+}