You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by ex...@apache.org on 2022/02/09 03:03:33 UTC
[nifi] branch main updated: NIFI-9621: Added Ignore Reserved Characters to FlattenJson
This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new bbc78f1 NIFI-9621: Added Ignore Reserved Characters to FlattenJson
bbc78f1 is described below
commit bbc78f154701941688ee227c6190cfcfdbc0848f
Author: Noel Baek <ja...@gmail.com>
AuthorDate: Sun Jan 23 23:20:38 2022 +0900
NIFI-9621: Added Ignore Reserved Characters to FlattenJson
- Upgraded to json-flattener from 0.12.0 to 0.13.0
This closes #5704
Signed-off-by: David Handermann <ex...@apache.org>
---
.../nifi/processors/standard/FlattenJson.java | 32 ++++++++--
.../processors/standard/TestFlattenJson.groovy | 70 ++++++++++++++++++++++
nifi-nar-bundles/nifi-standard-bundle/pom.xml | 2 +-
3 files changed, 97 insertions(+), 7 deletions(-)
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FlattenJson.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FlattenJson.java
index d85ea16..b7d5b24 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FlattenJson.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FlattenJson.java
@@ -123,6 +123,17 @@ public class FlattenJson extends AbstractProcessor {
.expressionLanguageSupported(ExpressionLanguageScope.NONE)
.build();
+ public static final PropertyDescriptor IGNORE_RESERVED_CHARACTERS = new PropertyDescriptor.Builder()
+ .name("ignore-reserved-characters")
+ .displayName("Ignore Reserved Characters")
+ .description("If true, reserved characters in keys will be ignored")
+ .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+ .allowableValues("true", "false")
+ .defaultValue("false")
+ .required(true)
+ .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ .build();
+
public static final PropertyDescriptor RETURN_TYPE = new PropertyDescriptor.Builder()
.name("flatten-json-return-type")
.displayName("Return Type")
@@ -171,6 +182,7 @@ public class FlattenJson extends AbstractProcessor {
List<PropertyDescriptor> props = new ArrayList<>();
props.add(SEPARATOR);
props.add(FLATTEN_MODE);
+ props.add(IGNORE_RESERVED_CHARACTERS);
props.add(RETURN_TYPE);
props.add(CHARACTER_SET);
props.add(PRETTY_PRINT);
@@ -207,6 +219,7 @@ public class FlattenJson extends AbstractProcessor {
final String returnType = context.getProperty(RETURN_TYPE).getValue();
final Charset charset = Charset.forName(context.getProperty(CHARACTER_SET).getValue());
final PrintMode printMode = context.getProperty(PRETTY_PRINT).asBoolean() ? PrintMode.PRETTY : PrintMode.MINIMAL;
+ final boolean ignoreReservedCharacters = context.getProperty(IGNORE_RESERVED_CHARACTERS).asBoolean();
try {
final StringBuilder contents = new StringBuilder();
@@ -214,12 +227,13 @@ public class FlattenJson extends AbstractProcessor {
final String resultedJson;
if (returnType.equals(RETURN_TYPE_FLATTEN)) {
- resultedJson = new JsonFlattener(contents.toString())
- .withFlattenMode(flattenMode)
- .withSeparator(separator)
- .withStringEscapePolicy(() -> StringEscapeUtils.ESCAPE_JSON)
- .withPrintMode(printMode)
- .flatten();
+ final JsonFlattener jsonFlattener = new JsonFlattener(contents.toString())
+ .withFlattenMode(flattenMode)
+ .withSeparator(separator)
+ .withStringEscapePolicy(() -> StringEscapeUtils.ESCAPE_JSON)
+ .withPrintMode(printMode);
+ setIgnoreReservedCharacters(ignoreReservedCharacters, jsonFlattener);
+ resultedJson = jsonFlattener.flatten();
} else {
resultedJson = new JsonUnflattener(contents.toString())
.withFlattenMode(flattenMode)
@@ -248,4 +262,10 @@ public class FlattenJson extends AbstractProcessor {
return FlattenMode.KEEP_ARRAYS;
}
}
+
+ private void setIgnoreReservedCharacters(final boolean ignoreReservedCharacters, final JsonFlattener jsonFlattener) {
+ if (ignoreReservedCharacters) {
+ jsonFlattener.ignoreReservedCharacters();
+ }
+ }
}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestFlattenJson.groovy b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestFlattenJson.groovy
index ce72c84..2485dc0 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestFlattenJson.groovy
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/groovy/org/apache/nifi/processors/standard/TestFlattenJson.groovy
@@ -368,4 +368,74 @@ class TestFlattenJson {
assert parsed.first.second.third[0] == ["one", "two", "three", "four", "five"]
}
}
+
+ @Test
+ void testFlattenWithIgnoreReservedCharacters() {
+ def testRunner = TestRunners.newTestRunner(FlattenJson.class)
+ def json = prettyPrint(toJson([
+ "first": [
+ "second.third": "Hello",
+ "fourth" : "World"
+ ]
+ ]))
+
+ testRunner.setProperty(FlattenJson.IGNORE_RESERVED_CHARACTERS, "true")
+
+ baseTest(testRunner, json, 2) { parsed ->
+ Assert.assertEquals("Separator not applied.", parsed["first.second.third"], "Hello")
+ Assert.assertEquals("Separator not applied.", parsed["first.fourth"], "World")
+ }
+ }
+
+ @Test
+ void testFlattenRecordSetWithIgnoreReservedCharacters() {
+ def testRunner = TestRunners.newTestRunner(FlattenJson.class)
+ def json = prettyPrint(toJson([
+ [
+ "first": [
+ "second_third": "Hello"
+ ]
+ ],
+ [
+ "first": [
+ "second_third": "World"
+ ]
+ ]
+ ]))
+ testRunner.setProperty(FlattenJson.SEPARATOR, "_")
+ testRunner.setProperty(FlattenJson.IGNORE_RESERVED_CHARACTERS, "true")
+
+ def expected = ["Hello", "World"]
+ baseTest(testRunner, json, 2) { parsed ->
+ Assert.assertTrue("Not a list", parsed instanceof List)
+ 0.upto(parsed.size() - 1) {
+ Assert.assertEquals("Missing values.", parsed[it]["first_second_third"], expected[it])
+ }
+ }
+ }
+
+ @Test
+ void testFlattenModeNormalWithIgnoreReservedCharacters() {
+ def testRunner = TestRunners.newTestRunner(FlattenJson.class)
+ def json = prettyPrint(toJson([
+ [
+ "first": [
+ "second_third": "Hello"
+ ]
+ ],
+ [
+ "first": [
+ "second_third": "World"
+ ]
+ ]
+ ]))
+ testRunner.setProperty(FlattenJson.SEPARATOR, "_")
+ testRunner.setProperty(FlattenJson.IGNORE_RESERVED_CHARACTERS, "true")
+ testRunner.setProperty(FlattenJson.FLATTEN_MODE, FlattenJson.FLATTEN_MODE_NORMAL)
+
+ baseTest(testRunner, json, 2) { parsed ->
+ Assert.assertEquals("Separator not applied.", "Hello", parsed["[0]_first_second_third"])
+ Assert.assertEquals("Separator not applied.", "World", parsed["[1]_first_second_third"])
+ }
+ }
}
diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/pom.xml
index 14ad736..5cb7e10 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml
@@ -300,7 +300,7 @@
<dependency>
<groupId>com.github.wnameless.json</groupId>
<artifactId>json-flattener</artifactId>
- <version>0.12.0</version>
+ <version>0.13.0</version>
</dependency>
<dependency>
<groupId>org.apache.bval</groupId>