You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by pv...@apache.org on 2017/03/23 08:04:28 UTC
nifi git commit: NIFI-3466 - Address non English locales in ParseCEF
Repository: nifi
Updated Branches:
refs/heads/master eb37f300c -> a298f1a70
NIFI-3466 - Address non English locales in ParseCEF
Signed-off-by: Pierre Villard <pi...@gmail.com>
This closes #1614.
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/a298f1a7
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/a298f1a7
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/a298f1a7
Branch: refs/heads/master
Commit: a298f1a7076a2a306a38e71bff1eb45da9614517
Parents: eb37f30
Author: Andre F de Miranda <tr...@users.noreply.github.com>
Authored: Wed Mar 22 23:20:18 2017 +1100
Committer: Pierre Villard <pi...@gmail.com>
Committed: Thu Mar 23 09:04:16 2017 +0100
----------------------------------------------------------------------
.../nifi-standard-processors/pom.xml | 2 +-
.../nifi/processors/standard/ParseCEF.java | 48 ++++++++++++-
.../nifi/processors/standard/TestParseCEF.java | 73 ++++++++++++++++++++
3 files changed, 120 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/a298f1a7/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 3be700a..f2e71a4 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -258,7 +258,7 @@ language governing permissions and limitations under the License. -->
<dependency>
<groupId>com.fluenda</groupId>
<artifactId>ParCEFone</artifactId>
- <version>1.2.0</version>
+ <version>1.2.2</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
http://git-wip-us.apache.org/repos/asf/nifi/blob/a298f1a7/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java
index 3417c0d..4107810 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseCEF.java
@@ -43,6 +43,9 @@ import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.annotation.documentation.Tags;
import org.apache.nifi.annotation.lifecycle.OnScheduled;
import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.components.Validator;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.processor.AbstractProcessor;
@@ -69,6 +72,7 @@ import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
@@ -146,6 +150,17 @@ public class ParseCEF extends AbstractProcessor {
.defaultValue(LOCAL_TZ)
.build();
+ public static final PropertyDescriptor DATETIME_REPRESENTATION = new PropertyDescriptor.Builder()
+ .name("DATETIME_REPRESENTATION")
+ .displayName("DateTime Locale")
+ .description("The IETF BCP 47 representation of the Locale to be used when parsing date " +
+ "fields with long or short month names (e.g. may <en-US> vs. mai. <fr-FR>. The default" +
+ "value is generally safe. Only change if having issues parsing CEF messages")
+ .required(true)
+ .addValidator(new ValidateLocale())
+ .defaultValue("en-US")
+ .build();
+
static final Relationship REL_FAILURE = new Relationship.Builder()
.name("failure")
.description("Any FlowFile that could not be parsed as a CEF message will be transferred to this Relationship without any attributes being added")
@@ -157,10 +172,11 @@ public class ParseCEF extends AbstractProcessor {
@Override
public List<PropertyDescriptor> getSupportedPropertyDescriptors() {
- final List<PropertyDescriptor>properties =new ArrayList<>();
+ final List<PropertyDescriptor>properties = new ArrayList<>();
properties.add(FIELDS_DESTINATION);
properties.add(APPEND_RAW_MESSAGE_TO_JSON);
properties.add(TIME_REPRESENTATION);
+ properties.add(DATETIME_REPRESENTATION);
return properties;
}
@@ -193,6 +209,7 @@ public class ParseCEF extends AbstractProcessor {
tzId = UTC;
break;
}
+
}
@Override
@@ -214,7 +231,11 @@ public class ParseCEF extends AbstractProcessor {
CommonEvent event;
try {
- event = parser.parse(buffer, true);
+ // parcefoneLocale defaults to en_US, so this should not fail. But we force failure in case the custom
+ // validator failed to identify an invalid Locale
+ final Locale parcefoneLocale = Locale.forLanguageTag(context.getProperty(DATETIME_REPRESENTATION).getValue());
+ event = parser.parse(buffer, true, parcefoneLocale);
+
} catch (Exception e) {
// This should never trigger but adding in here as a fencing mechanism to
// address possible ParCEFone bugs.
@@ -322,6 +343,29 @@ public class ParseCEF extends AbstractProcessor {
throws IOException, JsonProcessingException {
jsonGenerator.writeObject(macAddress.toString());
}
+ }
+
+ protected static class ValidateLocale implements Validator {
+ @Override
+ public ValidationResult validate(String subject, String input, ValidationContext context) {
+ if (null == input || input.isEmpty()) {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(false)
+ .explanation(subject + " cannot be empty").build();
+ }
+ final Locale testLocale = Locale.forLanguageTag(input);
+ final Locale[] availableLocales = Locale.getAvailableLocales();
+
+ // Check if the provided Locale is valid by checking against the first value of the array (i.e. "null" locale)
+ if (availableLocales[0].equals(testLocale)) {
+ // Locale matches the "null" locale so it is treated as invalid
+ return new ValidationResult.Builder().subject(subject).input(input).valid(false)
+ .explanation(input + " is not a valid locale format.").build();
+ } else {
+ return new ValidationResult.Builder().subject(subject).input(input).valid(true).build();
+
+ }
+
+ }
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/a298f1a7/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java
index 0da4d2b..5406613 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestParseCEF.java
@@ -28,6 +28,7 @@ import org.junit.Test;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.Locale;
import java.util.TimeZone;
@@ -239,6 +240,78 @@ public class TestParseCEF {
Assert.assertEquals(prettyEvent, extension.get("deviceCustomDate1").asText());
}
+ @Test
+ public void testNonEnglishDate() {
+ final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_ATTRIBUTES);
+ runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "fr-FR");
+ runner.assertValid();
+
+ String sample = sample1.replace("Feb", "f�vr.");
+ runner.enqueue(sample);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 1);
+ MockFlowFile mff1 = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+
+ mff1.assertAttributeEquals("cef.extension.rt", sdf.format(new Date(1423441663000L)));
+
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_ATTRIBUTES);
+ runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "et-EE");
+ runner.assertValid();
+
+ sample = sample1.replace("Feb", "veebr");
+ runner.enqueue(sample);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 2);
+ MockFlowFile mff2 = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+
+ mff2.assertAttributeEquals("cef.extension.rt", sdf.format(new Date(1423441663000L)));
+
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_ATTRIBUTES);
+ runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "ja-JP");
+ runner.assertValid();
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat("LLLL", Locale.forLanguageTag("ja-JP") );
+ String jpFeb = dateFormat.format(1423441663000L);
+
+ sample = sample1.replace("Feb", jpFeb );
+ runner.enqueue(sample);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(ParseCEF.REL_SUCCESS, 3);
+ MockFlowFile mff3 = runner.getFlowFilesForRelationship(ParseCEF.REL_SUCCESS).get(0);
+
+ mff3.assertAttributeEquals("cef.extension.rt", sdf.format(new Date(1423441663000L)));
+
+
+ }
+
+
+ @Test
+ public void testCustomValidator() {
+ final TestRunner runner = TestRunners.newTestRunner(new ParseCEF());
+ runner.setProperty(ParseCEF.FIELDS_DESTINATION, ParseCEF.DESTINATION_CONTENT);
+ runner.setProperty(ParseCEF.TIME_REPRESENTATION, ParseCEF.UTC);
+
+
+ runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "SPANGLISH");
+ runner.assertNotValid();
+ runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, "en-US");
+ runner.assertValid();
+
+
+ Locale availableLocales[] = Locale.getAvailableLocales();
+
+ for (Locale listedLocale : availableLocales ) {
+ if (!listedLocale.toString().isEmpty()) {
+ String input = listedLocale.toLanguageTag();
+ runner.setProperty(ParseCEF.DATETIME_REPRESENTATION, input );
+ runner.assertValid();
+ }
+ }
+ }
}