You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by jo...@apache.org on 2022/04/12 15:52:28 UTC
[nifi] 01/03: NIFI-9832: Fix disappearing XML element content when the element has attribute (#5896)
This is an automated email from the ASF dual-hosted git repository.
joewitt pushed a commit to branch support/nifi-1.16
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit b9458c0f926b77b589d3263dff2517be3888b3a8
Author: Peter Gyori <63...@users.noreply.github.com>
AuthorDate: Tue Apr 12 17:41:50 2022 +0200
NIFI-9832: Fix disappearing XML element content when the element has attribute (#5896)
- NIFI-9832: Additional test cases for XMLReader
---
.../nifi-record-serialization-services/pom.xml | 2 +
.../main/java/org/apache/nifi/xml/XMLReader.java | 12 +-
.../java/org/apache/nifi/xml/XMLRecordReader.java | 11 +-
.../apache/nifi/xml/inference/XmlRecordSource.java | 6 +-
.../additionalDetails.html | 351 +++++++++++++++++++++
.../org/apache/nifi/xml/TestInferXmlSchema.java | 11 +-
.../java/org/apache/nifi/xml/TestXMLReader.java | 218 ++++++++++---
.../test/resources/xml/field_with_sub-element.xml | 4 +
.../src/test/resources/xml/person_record.xml | 5 +
9 files changed, 561 insertions(+), 59 deletions(-)
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml
index e3ac2d4e2a..19b8997874 100755
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/pom.xml
@@ -218,6 +218,7 @@
<exclude>src/test/resources/syslog/syslog5424/log_mix.txt</exclude>
<exclude>src/test/resources/syslog/syslog5424/log_mix_in_error.txt</exclude>
<exclude>src/test/resources/text/testschema</exclude>
+ <exclude>src/test/resources/xml/field_with_sub-element.xml</exclude>
<exclude>src/test/resources/xml/people.xml</exclude>
<exclude>src/test/resources/xml/people2.xml</exclude>
<exclude>src/test/resources/xml/people3.xml</exclude>
@@ -236,6 +237,7 @@
<exclude>src/test/resources/xml/people_tag_in_characters.xml</exclude>
<exclude>src/test/resources/xml/people_with_header_and_comments.xml</exclude>
<exclude>src/test/resources/xml/person.xml</exclude>
+ <exclude>src/test/resources/xml/person_record.xml</exclude>
<exclude>src/test/resources/xml/testschema</exclude>
<exclude>src/test/resources/xml/testschema2</exclude>
<exclude>src/test/resources/xml/testschema3</exclude>
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLReader.java
index 52a9701507..20ee5195c6 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLReader.java
@@ -97,7 +97,10 @@ public class XMLReader extends SchemaRegistryService implements RecordReaderFact
.description("If tags with content (e. g. <field>content</field>) are defined as nested records in the schema, " +
"the name of the tag will be used as name for the record and the value of this property will be used as name for the field. " +
"If tags with content shall be parsed together with attributes (e. g. <field attribute=\"123\">content</field>), " +
- "they have to be defined as records. For additional information, see the section of processor usage.")
+ "they have to be defined as records. In such a case, the name of the tag will be used as the name for the record and " +
+ "the value of this property will be used as the name for the field holding the original content. The name of the attribute " +
+ "will be used to create a new record field, the content of which will be the value of the attribute. " +
+ "For more information, see the 'Additional Details...' section of the XMLReader controller service's documentation.")
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.required(false)
@@ -136,7 +139,12 @@ public class XMLReader extends SchemaRegistryService implements RecordReaderFact
@Override
protected SchemaAccessStrategy getSchemaAccessStrategy(final String strategy, final SchemaRegistry schemaRegistry, final PropertyContext context) {
- final RecordSourceFactory<XmlNode> sourceFactory = (variables, contentStream) -> new XmlRecordSource(contentStream, isMultipleRecords(context, variables));
+
+ final RecordSourceFactory<XmlNode> sourceFactory = (variables, contentStream) -> {
+ String contentFieldName = trim(context.getProperty(CONTENT_FIELD_NAME).evaluateAttributeExpressions(variables).getValue());
+ contentFieldName = (contentFieldName == null) ? "value" : contentFieldName;
+ return new XmlRecordSource(contentStream, contentFieldName, isMultipleRecords(context, variables));
+ };
final Supplier<SchemaInferenceEngine<XmlNode>> schemaInference = () -> new XmlSchemaInference(new TimeValueInference(dateFormat, timeFormat, timestampFormat));
return SchemaInferenceUtil.getSchemaAccessStrategy(strategy, context, getLogger(), sourceFactory, schemaInference,
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLRecordReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLRecordReader.java
index 2cb165b6e9..fd6d23b3e2 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLRecordReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/XMLRecordReader.java
@@ -339,8 +339,8 @@ public class XMLRecordReader implements RecordReader {
if (contentFieldName != null) {
recordValues.put(contentFieldName, content.toString());
} else {
- logger.debug("Found content for field that has to be parsed as record but property \"Field Name for Content\" is not set. " +
- "The content will not be added to the record.");
+ logger.debug("Found content for a field that was supposed to be named with the value of the \"Field Name for Content\" property but " +
+ "the property was not set. The content was not added to the record.");
}
return new MapRecord(new SimpleRecordSchema(Collections.emptyList()), recordValues);
@@ -486,10 +486,13 @@ public class XMLRecordReader implements RecordReader {
if (field.isPresent()) {
Object value = parseStringForType(content.toString(), contentFieldName, field.get().getDataType());
recordValues.put(contentFieldName, value);
+ } else {
+ logger.debug("Found content for a field that was supposed to be named with the value of the \"Field Name for Content\" property " +
+ "but no such field was present in the schema. The content was not added to the record.");
}
} else {
- logger.debug("Found content for field that is defined as record but property \"Field Name for Content\" is not set. " +
- "The content will not be added to record.");
+ logger.debug("Found content for a field that was supposed to be named with the value of the \"Field Name for Content\" property but " +
+ "the property was not set. The content was not added to the record.");
}
}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/inference/XmlRecordSource.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/inference/XmlRecordSource.java
index 3192e141aa..8352aed08d 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/inference/XmlRecordSource.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/java/org/apache/nifi/xml/inference/XmlRecordSource.java
@@ -35,8 +35,10 @@ import java.util.Map;
public class XmlRecordSource implements RecordSource<XmlNode> {
private final XMLEventReader xmlEventReader;
+ private final String contentFieldName;
- public XmlRecordSource(final InputStream in, final boolean ignoreWrapper) throws IOException {
+ public XmlRecordSource(final InputStream in, final String contentFieldName, final boolean ignoreWrapper) throws IOException {
+ this.contentFieldName = contentFieldName;
try {
final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
@@ -125,7 +127,7 @@ public class XmlRecordSource implements RecordSource<XmlNode> {
} else {
final String textContent = content.toString().trim();
if (!textContent.equals("")) {
- childNodes.put("value", new XmlTextNode("value", textContent));
+ childNodes.put(contentFieldName, new XmlTextNode(contentFieldName, textContent));
}
return new XmlContainerNode(nodeName, childNodes);
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/resources/docs/org.apache.nifi.xml.XMLReader/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/resources/docs/org.apache.nifi.xml.XMLReader/additionalDetails.html
index b8e1dff0d8..adcb3cc550 100755
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/resources/docs/org.apache.nifi.xml.XMLReader/additionalDetails.html
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/main/resources/docs/org.apache.nifi.xml.XMLReader/additionalDetails.html
@@ -286,6 +286,357 @@
for tags containing attributes and content.
</p>
+ <h2>Example: Tags with Attributes and Schema Inference</h2>
+
+ <p>
+ When the record's schema is not provided but inferred based on the data itself, providing a value for the "Field Name for Content" property
+ is especially important. (For detailed information on schema inference, see the "Schema Inference" section below.)
+ Let's focus on cases where an XML element (called <code><field_with_attribute></code> in the examples) has an XML attribute and some content and no sub-elements.
+ For the examples below, let's assume that a ConvertRecord processor is used, and it uses an XMLReader controller service and an XMLRecordSetWriter
+ controller service. The settings for XMLReader are provided separately for each example. The settings for XMLRecordSetWriter are common
+ for all the examples below. This way an XML to XML conversion is executed and comparing the input data with the output highlights
+ the schema inference behavior. The same behavior can be observed if a different Writer controller service is used.
+ XMLRecordSetWriter was chosen for these examples so that the input and the output are easily comparable.
+ The settings of the common XMLRecordSetWriter are the following:
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Schema Access Strategy</td>
+ <td><code>Inherit Record Schema</code></td>
+ </tr>
+ <tr>
+ <td>Suppress Null Values</td>
+ <td><code>Never Suppress</code></td>
+ </tr>
+ </table>
+
+ <h3>XML Attributes and Schema Inference Example 1</h3>
+
+ <p>
+ XMLReader settings:
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Schema Access Strategy</td>
+ <td><code>Infer Schema</code></td>
+ </tr>
+ <tr>
+ <td>Expect Records as Array</td>
+ <td><code>false</code></td>
+ </tr>
+ <tr>
+ <td>Field Name for Content</td>
+ <td>not set</td>
+ </tr>
+ </table>
+
+ <p>
+ Input:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute attr="attr_content">
+ content of field
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>As mentioned above, the element called "field_with_attribute" has an attribute and some content but no sub-element.</p>
+
+ <p>
+ Output:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute>
+ <attr>attr_content</attr>
+ <value></value>
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ In the XMLReader's settings, no value is set for the "Field Name for Content" property. In such cases the schema inference logic
+ adds a field named "value" to the schema. However, since "Field Name for Content" is not set, the data processing logic is instructed
+ not to consider the original content of the parent XML tags (<code><field_with_attribute></code> the content of which is "content of field"
+ in the example). So a new field named "value" appears in the schema but no value is assigned to it from the data, thus the field is empty.
+ The XML attribute (named "attr") is processed, a field named "attr" is added to the schema and the attribute's value ("attr_content") is assigned to it.
+ In a case like this, the parent field's original content is lost and a new field named "value" appears in the schema with no data assigned to it.
+ This is to make sure that no data is overwritten in the record if it already contains a field named "value". More on that case in Example 3 and Example 4.
+ </p>
+
+ <h3>XML Attributes and Schema Inference Example 2</h3>
+
+ <p>
+ In this example, the XMLReader's "Field Name for Content" property is filled with the value "original_content". The input data is the same as
+ in the previous example.
+ </p>
+
+ <p>
+ XMLReader settings:
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Schema Access Strategy</td>
+ <td><code>Infer Schema</code></td>
+ </tr>
+ <tr>
+ <td>Expect Records as Array</td>
+ <td><code>false</code></td>
+ </tr>
+ <tr>
+ <td>Field Name for Content</td>
+ <td><code>original_content</code></td>
+ </tr>
+ </table>
+
+ <p>
+ Input:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute attr="attr_content">
+ content of field
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ Output:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute>
+ <attr>attr_content</attr>
+ <original_content>content of field</original_content>
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ The XMLReader's "Field Name for Content" property contains the value "original_content" (the concrete value is not important, what is important
+ is that a value is provided and it does not clash with the name of any sub-element in <code><field_with_attribute></code>).
+ This explicitly tells the XMLReader controller service to create a field named "original_content" and make the original content of
+ the parent XML tag the value of the field named "original_content". Adding the XML attributed named "attr" works just like in the first example.
+ Since the <code><field_with_attribute></code> element had no child-element with the name "original_content", no data is lost.
+ </p>
+
+ <h3>XML Attributes and Schema Inference Example 3</h3>
+
+ <p>
+ In this example, XMLReader's "Field Name for Content" property is left empty. In the input data, the <code><field_with_attribute></code> element
+ has some content and a sub-element named <code><value></code>.
+ </p>
+
+ <p>
+ XMLReader settings:
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Schema Access Strategy</td>
+ <td><code>Infer Schema</code></td>
+ </tr>
+ <tr>
+ <td>Expect Records as Array</td>
+ <td><code>false</code></td>
+ </tr>
+ <tr>
+ <td>Field Name for Content</td>
+ <td>not set</td>
+ </tr>
+ </table>
+
+ <p>
+ Input:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute attr="attr_content">
+ content of field<value>123</value>
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ Output:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute>
+ <attr>attr_content</attr>
+ <value>123</value>
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ The "Field Name for Content" property is not set, and the XML element has a sub-element named "value". The name of the sub-element clashes with the
+ default field name added to the schema by the Schema Inference logic (see Example 1). As seen in the output data, the input XML attribute's value
+ is added to the record just like in the previous examples. The value of the <code><value></code> element is retained, but the content of the
+ <code><field_with_attribute></code> that was outside of the sub-element, is lost.
+ </p>
+
+ <h3>XML Attributes and Schema Inference Example 4</h3>
+
+ <p>
+ In this example, XMLReader's "Field Name for Content" property is given the value "value". In the input data, the <code><field_with_attribute></code> element
+ has some content and a sub-element named <code><value></code>. The name of the sub-element clashes with the value of the "Field Name for Content" property.
+ </p>
+
+ <p>
+ XMLReader settings:
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Schema Access Strategy</td>
+ <td><code>Infer Schema</code></td>
+ </tr>
+ <tr>
+ <td>Expect Records as Array</td>
+ <td><code>false</code></td>
+ </tr>
+ <tr>
+ <td>Field Name for Content</td>
+ <td><code>value</code></td>
+ </tr>
+ </table>
+
+ <p>
+ Input:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute attr="attr_content">
+ content of field<value>123</value>
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ Output:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute>
+ <attr>attr_content</attr>
+ <value>content of field</value>
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ The "Field Name for Content" property's value is "value", and the XML element has a sub-element named "value". The name of the sub-element clashes with the
+ value of the "Field Name for Content" property. The value of the <code><value></code> element is replaced by the content of the
+ <code><field_with_attribute></code> element, and the original content of the <code><value></code> element is lost.
+ </p>
+
+ <h3>XML Attributes and Schema Inference Example 5</h3>
+
+ <p>
+ To avoid losing any data, the XMLReader's "Field Name for Content" property needs to be given a value that does not clash with any sub-element's name
+ in the input data. In this example the input data is the same as in the previous one, but the "Field Name for Content" property's value is "original_content",
+ a value that does not clash with any sub-element name. No data is lost in this case.
+ </p>
+
+ <p>
+ XMLReader settings:
+ </p>
+
+ <table>
+ <tr>
+ <th>Property Name</th>
+ <th>Property Value</th>
+ </tr>
+ <tr>
+ <td>Schema Access Strategy</td>
+ <td><code>Infer Schema</code></td>
+ </tr>
+ <tr>
+ <td>Expect Records as Array</td>
+ <td><code>false</code></td>
+ </tr>
+ <tr>
+ <td>Field Name for Content</td>
+ <td><code>original_content</code></td>
+ </tr>
+ </table>
+
+ <p>
+ Input:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute attr="attr_content">
+ content of field<value>123</value>
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ Output:
+ </p>
+
+ <code>
+ <pre>
+ <record>
+ <field_with_attribute>
+ <attr>attr_content</attr>
+ <value>123</value>
+ <original_content>content of field</original_content>
+ </field_with_attribute>
+ </record></pre>
+ </code>
+
+ <p>
+ It can be seen in the output data, that the attribute has been added to the <code><field_with_attribute></code> element as a sub-element,
+ the <code><value></code> retained its value, and the original content of the <code><field_with_attribute></code> element has been added as a sub-element
+ named "original_content". This is because a value was chosen for the "Field Name for Content" property that does not clash with any of
+ the existing sub-elements of the input XML element (<code><field_with_attribute></code>). No data is lost.
+ </p>
+
<h2>Example: Array of records</h2>
<p>
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestInferXmlSchema.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestInferXmlSchema.java
index b5bdd76025..2a4cd14f10 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestInferXmlSchema.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestInferXmlSchema.java
@@ -93,7 +93,8 @@ public class TestInferXmlSchema {
@Test
public void testStringFieldWithAttributes() throws IOException {
- final RecordSchema schema = inferSchema("src/test/resources/xml/TextNodeWithAttribute.xml", true);
+ final String contentFieldName = "contentfield";
+ final RecordSchema schema = inferSchema("src/test/resources/xml/TextNodeWithAttribute.xml", contentFieldName, true);
assertEquals(3, schema.getFieldCount());
@@ -106,12 +107,16 @@ public class TestInferXmlSchema {
final RecordSchema childSchema = ((RecordDataType) softwareDataType).getChildSchema();
assertSame(RecordFieldType.BOOLEAN, childSchema.getDataType("favorite").get().getFieldType());
- assertSame(RecordFieldType.STRING, childSchema.getDataType("value").get().getFieldType());
+ assertSame(RecordFieldType.STRING, childSchema.getDataType(contentFieldName).get().getFieldType());
}
private RecordSchema inferSchema(final String filename, final boolean ignoreWrapper) throws IOException {
+ return inferSchema(filename, "contentfield", ignoreWrapper);
+ }
+
+ private RecordSchema inferSchema(final String filename, final String contentFieldName, final boolean ignoreWrapper) throws IOException {
final File file = new File(filename);
- final RecordSourceFactory<XmlNode> xmlSourceFactory = (var, in) -> new XmlRecordSource(in, ignoreWrapper);
+ final RecordSourceFactory<XmlNode> xmlSourceFactory = (var, in) -> new XmlRecordSource(in, contentFieldName, ignoreWrapper);
final SchemaInferenceEngine<XmlNode> schemaInference = new XmlSchemaInference(timeValueInference);
final InferSchemaAccessStrategy<XmlNode> inferStrategy = new InferSchemaAccessStrategy<>(xmlSourceFactory, schemaInference, Mockito.mock(ComponentLog.class));
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestXMLReader.java b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestXMLReader.java
index 5de7eac402..6904a6f599 100644
--- a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestXMLReader.java
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/xml/TestXMLReader.java
@@ -17,8 +17,10 @@
package org.apache.nifi.xml;
+import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.schema.access.SchemaAccessUtils;
+import org.apache.nifi.schema.inference.SchemaInferenceUtil;
import org.apache.nifi.util.MockFlowFile;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
@@ -31,43 +33,48 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import static junit.framework.TestCase.assertEquals;
public class TestXMLReader {
- private XMLReader reader;
-
private final String ATTRIBUTE_PREFIX = "attribute_prefix";
private final String CONTENT_NAME = "content_field";
private final String EVALUATE_IS_ARRAY = "xml.stream.is.array";
- public TestRunner setup(String filePath) throws InitializationException, IOException {
-
+ private TestRunner setup(Map<PropertyDescriptor, String> xmlReaderProperties) throws InitializationException {
TestRunner runner = TestRunners.newTestRunner(TestXMLReaderProcessor.class);
- reader = new XMLReader();
+ XMLReader reader = new XMLReader();
+
runner.addControllerService("xml_reader", reader);
runner.setProperty(TestXMLReaderProcessor.XML_READER, "xml_reader");
- final String outputSchemaText = new String(Files.readAllBytes(Paths.get(filePath)));
- runner.setProperty(reader, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
- runner.setProperty(reader, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
+ for (Map.Entry<PropertyDescriptor, String> entry : xmlReaderProperties.entrySet()) {
+ runner.setProperty(reader, entry.getKey(), entry.getValue());
+ }
+ runner.enableControllerService(reader);
return runner;
}
@Test
- public void testRecordFormat() throws IOException, InitializationException {
- TestRunner runner = setup("src/test/resources/xml/testschema");
+ public void testRecordFormatDeterminedBasedOnAttribute() throws IOException, InitializationException {
+ String outputSchemaPath = "src/test/resources/xml/testschema";
+ String outputSchemaText = new String(Files.readAllBytes(Paths.get(outputSchemaPath)));
- runner.setProperty(reader, XMLReader.RECORD_FORMAT, XMLReader.RECORD_EVALUATE);
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY.getValue());
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_EVALUATE.getValue());
+ TestRunner runner = setup(xmlReaderProperties);
- runner.enableControllerService(reader);
-
- InputStream is = new FileInputStream("src/test/resources/xml/people.xml");
- runner.enqueue(is, Collections.singletonMap(EVALUATE_IS_ARRAY, "true"));
- runner.run();
+ try (InputStream is = new FileInputStream("src/test/resources/xml/people.xml")) {
+ runner.enqueue(is, Collections.singletonMap(EVALUATE_IS_ARRAY, "true"));
+ runner.run();
+ }
List<MockFlowFile> flowFile = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS);
List<String> records = Arrays.asList((new String(runner.getContentAsByteArray(flowFile.get(0)))).split("\n"));
@@ -76,16 +83,20 @@ public class TestXMLReader {
}
@Test
- public void testRecordFormat2() throws IOException, InitializationException {
- TestRunner runner = setup("src/test/resources/xml/testschema");
+ public void testRecordFormatArray() throws IOException, InitializationException {
+ String outputSchemaPath = "src/test/resources/xml/testschema";
+ String outputSchemaText = new String(Files.readAllBytes(Paths.get(outputSchemaPath)));
- runner.setProperty(reader, XMLReader.RECORD_FORMAT, XMLReader.RECORD_ARRAY);
-
- runner.enableControllerService(reader);
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY.getValue());
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_ARRAY.getValue());
+ TestRunner runner = setup(xmlReaderProperties);
- InputStream is = new FileInputStream("src/test/resources/xml/people.xml");
- runner.enqueue(is, Collections.singletonMap(EVALUATE_IS_ARRAY, "true"));
- runner.run();
+ try (InputStream is = new FileInputStream("src/test/resources/xml/people.xml")) {
+ runner.enqueue(is, Collections.singletonMap(EVALUATE_IS_ARRAY, "true"));
+ runner.run();
+ }
List<MockFlowFile> flowFile = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS);
List<String> records = Arrays.asList((new String(runner.getContentAsByteArray(flowFile.get(0)))).split("\n"));
@@ -94,16 +105,20 @@ public class TestXMLReader {
}
@Test
- public void testRecordFormat3() throws IOException, InitializationException {
- TestRunner runner = setup("src/test/resources/xml/testschema");
+ public void testRecordFormatNotArray() throws IOException, InitializationException {
+ String outputSchemaPath = "src/test/resources/xml/testschema";
+ String outputSchemaText = new String(Files.readAllBytes(Paths.get(outputSchemaPath)));
- runner.setProperty(reader, XMLReader.RECORD_FORMAT, XMLReader.RECORD_SINGLE);
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY.getValue());
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_SINGLE.getValue());
+ TestRunner runner = setup(xmlReaderProperties);
- runner.enableControllerService(reader);
-
- InputStream is = new FileInputStream("src/test/resources/xml/person.xml");
- runner.enqueue(is, Collections.singletonMap(EVALUATE_IS_ARRAY, "true"));
- runner.run();
+ try (InputStream is = new FileInputStream("src/test/resources/xml/person.xml")) {
+ runner.enqueue(is, Collections.singletonMap(EVALUATE_IS_ARRAY, "true"));
+ runner.run();
+ }
List<MockFlowFile> flowFile = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS);
List<String> records = Arrays.asList(new String(runner.getContentAsByteArray(flowFile.get(0))).split("\n"));
@@ -113,16 +128,20 @@ public class TestXMLReader {
@Test
public void testAttributePrefix() throws IOException, InitializationException {
- TestRunner runner = setup("src/test/resources/xml/testschema");
+ String outputSchemaPath = "src/test/resources/xml/testschema";
+ String outputSchemaText = new String(Files.readAllBytes(Paths.get(outputSchemaPath)));
- runner.setProperty(reader, XMLReader.ATTRIBUTE_PREFIX, "${" + ATTRIBUTE_PREFIX + "}");
- runner.setProperty(reader, XMLReader.RECORD_FORMAT, XMLReader.RECORD_ARRAY);
-
- runner.enableControllerService(reader);
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY.getValue());
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
+ xmlReaderProperties.put(XMLReader.ATTRIBUTE_PREFIX, "${" + ATTRIBUTE_PREFIX + "}");
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_ARRAY.getValue());
+ TestRunner runner = setup(xmlReaderProperties);
- InputStream is = new FileInputStream("src/test/resources/xml/people.xml");
- runner.enqueue(is, Collections.singletonMap(ATTRIBUTE_PREFIX, "ATTR_"));
- runner.run();
+ try (InputStream is = new FileInputStream("src/test/resources/xml/people.xml")) {
+ runner.enqueue(is, Collections.singletonMap(ATTRIBUTE_PREFIX, "ATTR_"));
+ runner.run();
+ }
List<MockFlowFile> flowFile = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS);
List<String> records = Arrays.asList(new String(runner.getContentAsByteArray(flowFile.get(0))).split("\n"));
@@ -136,16 +155,20 @@ public class TestXMLReader {
@Test
public void testContentField() throws IOException, InitializationException {
- TestRunner runner = setup("src/test/resources/xml/testschema2");
+ String outputSchemaPath = "src/test/resources/xml/testschema2";
+ String outputSchemaText = new String(Files.readAllBytes(Paths.get(outputSchemaPath)));
- runner.setProperty(reader, XMLReader.CONTENT_FIELD_NAME, "${" + CONTENT_NAME + "}");
- runner.setProperty(reader, XMLReader.RECORD_FORMAT, XMLReader.RECORD_ARRAY);
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY.getValue());
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
+ xmlReaderProperties.put(XMLReader.CONTENT_FIELD_NAME, "${" + CONTENT_NAME + "}");
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_ARRAY.getValue());
+ TestRunner runner = setup(xmlReaderProperties);
- runner.enableControllerService(reader);
-
- InputStream is = new FileInputStream("src/test/resources/xml/people_tag_in_characters.xml");
- runner.enqueue(is, Collections.singletonMap(CONTENT_NAME, "CONTENT"));
- runner.run();
+ try (InputStream is = new FileInputStream("src/test/resources/xml/people_tag_in_characters.xml")) {
+ runner.enqueue(is, Collections.singletonMap(CONTENT_NAME, "CONTENT"));
+ runner.run();
+ }
List<MockFlowFile> flowFile = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS);
List<String> records = Arrays.asList(new String(runner.getContentAsByteArray(flowFile.get(0))).split("\n"));
@@ -157,4 +180,103 @@ public class TestXMLReader {
assertEquals("MapRecord[{ID=P4, NAME=MapRecord[{CONTENT=Elenora Scrivens, ATTR=attr content, INNER=inner content}], AGE=16}]", records.get(3));
assertEquals("MapRecord[{ID=P5, NAME=MapRecord[{INNER=inner content}]}]", records.get(4));
}
+
+ @Test
+ public void testInferSchema() throws InitializationException, IOException {
+ String expectedContent = "MapRecord[{software=MapRecord[{" + CONTENT_NAME + "=Apache NiFi, favorite=true}], num=123, name=John Doe}]";
+
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaInferenceUtil.INFER_SCHEMA.getValue());
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_SINGLE.getValue());
+ xmlReaderProperties.put(XMLReader.CONTENT_FIELD_NAME, CONTENT_NAME);
+ TestRunner runner = setup(xmlReaderProperties);
+
+ try (InputStream is = new FileInputStream("src/test/resources/xml/person_record.xml")) {
+ runner.enqueue(is);
+ runner.run();
+ }
+
+ MockFlowFile out = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS).get(0);
+ String actualContent = out.getContent();
+ assertEquals(expectedContent, actualContent);
+ }
+
+ @Test
+ public void testInferSchemaContentFieldNameNotSet() throws InitializationException, IOException {
+ String expectedContent = "MapRecord[{software=MapRecord[{favorite=true}], num=123, name=John Doe}]";
+
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaInferenceUtil.INFER_SCHEMA.getValue());
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_SINGLE.getValue());
+ TestRunner runner = setup(xmlReaderProperties);
+
+ try (InputStream is = new FileInputStream("src/test/resources/xml/person_record.xml")) {
+ runner.enqueue(is);
+ runner.run();
+ }
+
+ MockFlowFile out = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS).get(0);
+ String actualContent = out.getContent();
+ assertEquals(expectedContent, actualContent);
+ }
+
+ @Test
+ public void testInferSchemaContentFieldNameNotSetSubElementExists() throws InitializationException, IOException {
+ String expectedContent = "MapRecord[{field_with_attribute=MapRecord[{attr=attr_content, value=123}]}]";
+
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaInferenceUtil.INFER_SCHEMA.getValue());
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_SINGLE.getValue());
+ TestRunner runner = setup(xmlReaderProperties);
+
+ try (InputStream is = new FileInputStream("src/test/resources/xml/field_with_sub-element.xml")) {
+ runner.enqueue(is);
+ runner.run();
+ }
+
+ MockFlowFile out = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS).get(0);
+ String actualContent = out.getContent();
+ assertEquals(expectedContent, actualContent);
+ }
+
+ @Test
+ public void testInferSchemaContentFieldNameSetSubElementExistsNameClash() throws InitializationException, IOException {
+ String expectedContent = "MapRecord[{field_with_attribute=MapRecord[{attr=attr_content, value=content of field}]}]";
+
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaInferenceUtil.INFER_SCHEMA.getValue());
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_SINGLE.getValue());
+ xmlReaderProperties.put(XMLReader.CONTENT_FIELD_NAME, "value");
+ TestRunner runner = setup(xmlReaderProperties);
+
+ try (InputStream is = new FileInputStream("src/test/resources/xml/field_with_sub-element.xml")) {
+ runner.enqueue(is);
+ runner.run();
+ }
+
+ MockFlowFile out = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS).get(0);
+ String actualContent = out.getContent();
+ assertEquals(expectedContent, actualContent);
+ }
+
+ @Test
+ public void testInferSchemaContentFieldNameSetSubElementExistsNoNameClash() throws InitializationException, IOException {
+ String expectedContent = "MapRecord[{field_with_attribute=MapRecord[{" +CONTENT_NAME + "=content of field, " +
+ "attr=attr_content, value=123}]}]";
+
+ Map<PropertyDescriptor, String> xmlReaderProperties = new HashMap<>();
+ xmlReaderProperties.put(SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaInferenceUtil.INFER_SCHEMA.getValue());
+ xmlReaderProperties.put(XMLReader.RECORD_FORMAT, XMLReader.RECORD_SINGLE.getValue());
+ xmlReaderProperties.put(XMLReader.CONTENT_FIELD_NAME, CONTENT_NAME);
+ TestRunner runner = setup(xmlReaderProperties);
+
+ try (InputStream is = new FileInputStream("src/test/resources/xml/field_with_sub-element.xml")) {
+ runner.enqueue(is);
+ runner.run();
+ }
+
+ MockFlowFile out = runner.getFlowFilesForRelationship(TestXMLReaderProcessor.SUCCESS).get(0);
+ String actualContent = out.getContent();
+ assertEquals(expectedContent, actualContent);
+ }
}
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/field_with_sub-element.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/field_with_sub-element.xml
new file mode 100644
index 0000000000..2c9146119d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/field_with_sub-element.xml
@@ -0,0 +1,4 @@
+<record>
+ <field_with_attribute attr="attr_content">content of field<value>123</value>
+ </field_with_attribute>
+</record>
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/person_record.xml b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/person_record.xml
new file mode 100644
index 0000000000..08b39093e0
--- /dev/null
+++ b/nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/resources/xml/person_record.xml
@@ -0,0 +1,5 @@
+<record>
+ <num>123</num>
+ <name>John Doe</name>
+ <software favorite="true">Apache NiFi</software>
+</record>
\ No newline at end of file