You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by pv...@apache.org on 2022/01/29 12:57:04 UTC
[nifi] branch main updated: NIFI-9629: Ensure that when we are setting default values on Avro GenericRecord objects that we convert from the schema's default value to the proper type
This is an automated email from the ASF dual-hosted git repository.
pvillard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 038811d NIFI-9629: Ensure that when we are setting default values on Avro GenericRecord objects that we convert from the schema's default value to the proper type
038811d is described below
commit 038811d0dd8d317b6cfe0cffd3f1f4be78753250
Author: Mark Payne <ma...@hotmail.com>
AuthorDate: Tue Jan 25 16:16:27 2022 -0500
NIFI-9629: Ensure that when we are setting default values on Avro GenericRecord objects that we convert from the schema's default value to the proper type
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #5716.
---
.../main/java/org/apache/nifi/avro/AvroTypeUtil.java | 8 ++++++--
.../java/org/apache/nifi/avro/TestAvroTypeUtil.java | 18 +++++++++++++++++-
2 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
index d8d6e10..36c62a5 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/main/java/org/apache/nifi/avro/AvroTypeUtil.java
@@ -622,12 +622,16 @@ public class AvroTypeUtil {
// see if the Avro schema has any fields that aren't in the RecordSchema, and if those fields have a default
// value then we want to populate it in the GenericRecord being produced
for (final Field field : avroSchema.getFields()) {
- if (field.defaultVal() == null) {
+ final Object defaultValue = field.defaultVal();
+ if (defaultValue == null || defaultValue == JsonProperties.NULL_VALUE) {
continue;
}
if (rec.get(field.name()) == null) {
- rec.put(field.name(), field.defaultVal());
+ // The default value may not actually be the proper value for Avro. For example, the schema may indicate that we need a long but provide a default value of 0.
+ // To address this, we need to ensure that the value that we set is correct based on the Avro schema, so we need to call convertToAvroObject even on the default value.
+ final Object normalized = convertToAvroObject(defaultValue, field.schema());
+ rec.put(field.name(), normalized);
}
}
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
index dac8024..5f2204b 100644
--- a/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-avro-record-utils/src/test/java/org/apache/nifi/avro/TestAvroTypeUtil.java
@@ -139,9 +139,25 @@ public class TestAvroTypeUtil {
final GenericRecord avroRecord = AvroTypeUtil.createAvroRecord(record, avroSchema);
assertEquals("John Doe", avroRecord.get("name"));
assertEquals("blue", avroRecord.get("color"));
-
}
+ @Test
+ public void testAvroDefaultedLong() throws IOException {
+ final List<RecordField> fields = new ArrayList<>();
+ fields.add(new RecordField("name", RecordFieldType.STRING.getDataType()));
+ final RecordSchema personSchema = new SimpleRecordSchema(fields);
+
+ final org.apache.nifi.serialization.record.Record record = new MapRecord(personSchema, Collections.singletonMap("name", "John Doe"));
+ final Schema avroSchema = SchemaBuilder.record("person").namespace("nifi")
+ .fields()
+ .requiredString("name")
+ .name("number").type().longType().longDefault(0)
+ .endRecord();
+
+ final GenericRecord avroRecord = AvroTypeUtil.createAvroRecord(record, avroSchema);
+ assertEquals("John Doe", avroRecord.get("name"));
+ assertEquals(0L, avroRecord.get("number"));
+ }
@Test
public void testCreateAvroSchemaPrimitiveTypes() {