You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by bb...@apache.org on 2018/07/16 15:51:23 UTC
[3/3] nifi git commit: NIFI-5337 Syslog 5424 Record Reader and
nifi-syslog-utils
NIFI-5337 Syslog 5424 Record Reader and nifi-syslog-utils
- Create nifi-syslog-utils to move syslog parsing functionalty to a central location shared by the processors and serialization/record system.
- Refactor Processors to use these utils
- Update 5424 syslog classes using simple-syslog-5424 to pick up new changes to support this work, as well as keep dependencies/types from bleeding out to the
processors or readers
- Refactor Syslog5424Event and Parser
- Create Syslog5424RecordReader
- per review, handle blank message differently from eof
- name schema per review
This closes #2816.
Signed-off-by: Bryan Bende <bb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b1022043
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b1022043
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b1022043
Branch: refs/heads/master
Commit: b10220439cfddd876e01f1a2279b33552e47c7ac
Parents: ec7f131
Author: Otto Fowler <ot...@gmail.com>
Authored: Fri Jun 22 15:44:38 2018 -0400
Committer: Bryan Bende <bb...@apache.org>
Committed: Mon Jul 16 11:50:55 2018 -0400
----------------------------------------------------------------------
.../nifi-syslog-utils/pom.xml | 48 ++++
.../syslog/attributes/Syslog5424Attributes.java | 50 ++++
.../syslog/attributes/SyslogAttributes.java | 60 +++++
.../nifi/syslog/events/Syslog5424Event.java | 116 +++++++++
.../apache/nifi/syslog/events/SyslogEvent.java | 180 +++++++++++++
.../syslog/keyproviders/SimpleKeyProvider.java | 105 ++++++++
.../keyproviders/SyslogPrefixedKeyProvider.java | 105 ++++++++
.../syslog/parsers/StrictSyslog5424Parser.java | 130 ++++++++++
.../nifi/syslog/parsers/SyslogParser.java | 175 +++++++++++++
.../syslog/utils/NifiStructuredDataPolicy.java | 35 +++
.../nifi/syslog/utils/NilHandlingPolicy.java | 36 +++
.../syslog/BaseStrictSyslog5424ParserTest.java | 206 +++++++++++++++
.../StrictSyslog5424ParserDashPolicyTest.java | 26 ++
.../StrictSyslog5424ParserNullPolicyTest.java | 26 ++
.../StrictSyslog5424ParserOmitPolicyTest.java | 26 ++
.../apache/nifi/syslog/TestSyslogParser.java | 255 +++++++++++++++++++
nifi-nar-bundles/nifi-extension-utils/pom.xml | 1 +
.../nifi-standard-processors/pom.xml | 9 +-
.../nifi/processors/standard/ListenSyslog.java | 30 +--
.../nifi/processors/standard/ParseSyslog.java | 39 +--
.../processors/standard/ParseSyslog5424.java | 32 ++-
.../nifi/processors/standard/PutSyslog.java | 2 +-
.../standard/syslog/StrictSyslog5424Parser.java | 206 ---------------
.../standard/syslog/Syslog5424Attributes.java | 43 ----
.../standard/syslog/Syslog5424Event.java | 103 --------
.../standard/syslog/SyslogAttributes.java | 48 ----
.../processors/standard/syslog/SyslogEvent.java | 180 -------------
.../standard/syslog/SyslogParser.java | 173 -------------
.../standard/ITListenSyslogGroovy.groovy | 2 +-
.../standard/ParseSyslogGroovyTest.groovy | 2 +-
.../processors/standard/ITListenSyslog.java | 22 +-
.../processors/standard/TestListenSyslog.java | 12 +-
.../processors/standard/TestParseSyslog.java | 38 +--
.../standard/TestParseSyslog5424.java | 8 +-
.../util/BaseStrictSyslog5424ParserTest.java | 205 ---------------
.../StrictSyslog5424ParserDashPolicyTest.java | 25 --
.../StrictSyslog5424ParserNullPolicyTest.java | 25 --
.../StrictSyslog5424ParserOmitPolicyTest.java | 25 --
.../standard/util/TestSyslogParser.java | 255 -------------------
nifi-nar-bundles/nifi-standard-bundle/pom.xml | 5 -
.../src/main/resources/META-INF/NOTICE | 8 +
.../nifi-record-serialization-services/pom.xml | 9 +
.../apache/nifi/syslog/Syslog5424Reader.java | 155 +++++++++++
.../nifi/syslog/Syslog5424RecordReader.java | 121 +++++++++
...org.apache.nifi.controller.ControllerService | 2 +-
.../additionalDetails.html | 91 +++++++
.../nifi/syslog/TestSyslog5424RecordReader.java | 223 ++++++++++++++++
.../test/resources/syslog/syslog5424/log.txt | 1 +
.../resources/syslog/syslog5424/log_all.txt | 1 +
.../resources/syslog/syslog5424/log_mix.txt | 3 +
.../syslog/syslog5424/log_mix_in_error.txt | 4 +
51 files changed, 2302 insertions(+), 1385 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/pom.xml
new file mode 100644
index 0000000..dd94370
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/pom.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-extension-utils</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>nifi-syslog-utils</artifactId>
+ <packaging>jar</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>com.github.palindromicity</groupId>
+ <artifactId>simple-syslog-5424</artifactId>
+ <version>0.0.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-mock</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/Syslog5424Attributes.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/Syslog5424Attributes.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/Syslog5424Attributes.java
new file mode 100644
index 0000000..346bd7a
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/Syslog5424Attributes.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog.attributes;
+
+import org.apache.nifi.flowfile.attributes.FlowFileAttributeKey;
+
+/**
+ * FlowFile Attributes for each Syslog message.
+ */
+public enum Syslog5424Attributes implements FlowFileAttributeKey {
+
+ SYSLOG_APP_NAME("syslog.appName"),
+ SYSLOG_PROCID("syslog.procid"),
+ SYSLOG_MESSAGEID("syslog.messageid"),
+ SYSLOG_STRUCTURED_BASE("syslog.structuredData"),
+ SYSLOG_STRUCTURED_ELEMENT_ID_FMT("syslog.structuredData.%s"),
+ SYSLOG_STRUCTURED_ELEMENT_ID_PNAME_FMT("syslog.structuredData.%s.%s"),
+ SYSLOG_STRUCTURED_ELEMENT_ID_PNAME_PATTERN("syslog.structuredData\\.(.*)\\.(.*)$"),
+ APP_NAME("appName"),
+ PROCID("procid"),
+ MESSAGEID("messageid"),
+ STRUCTURED_BASE("structuredData"),
+ STRUCTURED_ELEMENT_ID_FMT("structuredData.%s"),
+ STRUCTURED_ELEMENT_ID_PNAME_FMT("structuredData.%s.%s"),
+ STRUCTURED_ELEMENT_ID_PNAME_PATTERN("structuredData\\.(.*)\\.(.*)$");
+ private String key;
+
+ Syslog5424Attributes(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public String key() {
+ return key;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/SyslogAttributes.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/SyslogAttributes.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/SyslogAttributes.java
new file mode 100644
index 0000000..1eb2dc8
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/attributes/SyslogAttributes.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog.attributes;
+
+import org.apache.nifi.flowfile.attributes.FlowFileAttributeKey;
+
+/**
+ * FlowFile Attributes for each Syslog message.
+ */
+public enum SyslogAttributes implements FlowFileAttributeKey {
+
+ SYSLOG_PRIORITY("syslog.priority"),
+ SYSLOG_SEVERITY("syslog.severity"),
+ SYSLOG_FACILITY("syslog.facility"),
+ SYSLOG_VERSION("syslog.version"),
+ SYSLOG_TIMESTAMP("syslog.timestamp"),
+ SYSLOG_HOSTNAME("syslog.hostname"),
+ SYSLOG_SENDER("syslog.sender"),
+ SYSLOG_BODY("syslog.body"),
+ SYSLOG_VALID("syslog.valid"),
+ SYSLOG_PROTOCOL("syslog.protocol"),
+ SYSLOG_PORT("syslog.port"),
+
+ PRIORITY("priority"),
+ SEVERITY("severity"),
+ FACILITY("facility"),
+ VERSION("version"),
+ TIMESTAMP("timestamp"),
+ HOSTNAME("hostname"),
+ SENDER("sender"),
+ BODY("body"),
+ VALID("valid"),
+ PROTOCOL("protocol"),
+ PORT("port");
+
+ private String key;
+
+ SyslogAttributes(String key) {
+ this.key = key;
+ }
+
+ @Override
+ public String key() {
+ return key;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/Syslog5424Event.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/Syslog5424Event.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/Syslog5424Event.java
new file mode 100644
index 0000000..7fb00bb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/Syslog5424Event.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog.events;
+
+import java.util.Map;
+
+/**
+ * Encapsulates the parsed information for a single Syslog 5424 event.
+ */
+public class Syslog5424Event {
+ private final Map<String, Object> fieldMap;
+ private final String fullMessage;
+ private final byte[] rawMessage;
+ private final String sender;
+ private final boolean valid;
+ private final Exception exception;
+
+ private Syslog5424Event(final Builder builder) {
+ this.fieldMap = builder.fieldMap;
+ this.fullMessage = builder.fullMessage;
+ this.rawMessage = builder.rawMessage;
+ this.sender = builder.sender;
+ this.valid = builder.valid;
+ this.exception = builder.exception;
+ }
+
+ public Map<String, Object> getFieldMap() {
+ return fieldMap;
+ }
+
+ public Exception getException() {
+ return exception;
+ }
+
+ public String getFullMessage() {
+ return fullMessage;
+ }
+
+ public byte[] getRawMessage() {
+ return rawMessage;
+ }
+
+ public String getSender() {
+ return sender;
+ }
+
+ public boolean isValid() {
+ return valid;
+ }
+
+ public static final class Builder {
+ private Exception exception;
+ private String fullMessage;
+ private String sender;
+ private Map<String, Object> fieldMap;
+ private byte[] rawMessage;
+ private boolean valid;
+
+ public void reset() {
+ this.fieldMap = null;
+ this.sender = null;
+ this.fullMessage = null;
+ this.valid = false;
+ this.exception = null;
+ }
+
+ public Builder sender(String sender) {
+ this.sender = sender;
+ return this;
+ }
+
+ public Builder exception(Exception exception) {
+ this.exception = exception;
+ return this;
+ }
+
+ public Builder fieldMap(Map<String, Object> fieldMap) {
+ this.fieldMap = fieldMap;
+ return this;
+ }
+
+ public Builder fullMessage(String fullMessage) {
+ this.fullMessage = fullMessage;
+ return this;
+ }
+
+ public Builder rawMessage(byte[] rawMessage) {
+ this.rawMessage = rawMessage;
+ return this;
+ }
+
+ public Builder valid(boolean valid) {
+ this.valid = valid;
+ return this;
+ }
+
+ public Syslog5424Event build() {
+ return new Syslog5424Event(this);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/SyslogEvent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/SyslogEvent.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/SyslogEvent.java
new file mode 100644
index 0000000..a6f711c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/events/SyslogEvent.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog.events;
+
+/**
+ * Encapsulates the parsed information for a single Syslog event.
+ */
+public class SyslogEvent {
+
+ private final String priority;
+ private final String severity;
+ private final String facility;
+ private final String version;
+ private final String timeStamp;
+ private final String hostName;
+ private final String sender;
+ private final String msgBody;
+ private final String fullMessage;
+ private final byte[] rawMessage;
+ private final boolean valid;
+
+ private SyslogEvent(final Builder builder) {
+ this.priority = builder.priority;
+ this.severity = builder.severity;
+ this.facility = builder.facility;
+ this.version = builder.version;
+ this.timeStamp = builder.timeStamp;
+ this.hostName = builder.hostName;
+ this.sender = builder.sender;
+ this.msgBody = builder.msgBody;
+ this.fullMessage = builder.fullMessage;
+ this.rawMessage = builder.rawMessage;
+ this.valid = builder.valid;
+ }
+
+ public String getPriority() {
+ return priority;
+ }
+
+ public String getSeverity() {
+ return severity;
+ }
+
+ public String getFacility() {
+ return facility;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public String getTimeStamp() {
+ return timeStamp;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
+ public String getSender() {
+ return sender;
+ }
+
+ public String getMsgBody() {
+ return msgBody;
+ }
+
+ public String getFullMessage() {
+ return fullMessage;
+ }
+
+ public byte[] getRawMessage() {
+ return rawMessage;
+ }
+
+ public boolean isValid() {
+ return valid;
+ }
+
+ public static final class Builder {
+ private String priority;
+ private String severity;
+ private String facility;
+ private String version;
+ private String timeStamp;
+ private String hostName;
+ private String sender;
+ private String msgBody;
+ private String fullMessage;
+ private byte[] rawMessage;
+ private boolean valid;
+
+ public void reset() {
+ this.priority = null;
+ this.severity = null;
+ this.facility = null;
+ this.version = null;
+ this.timeStamp = null;
+ this.hostName = null;
+ this.sender = null;
+ this.msgBody = null;
+ this.fullMessage = null;
+ this.valid = false;
+ }
+
+ public Builder priority(String priority) {
+ this.priority = priority;
+ return this;
+ }
+
+ public Builder severity(String severity) {
+ this.severity = severity;
+ return this;
+ }
+
+ public Builder facility(String facility) {
+ this.facility = facility;
+ return this;
+ }
+
+ public Builder version(String version) {
+ this.version = version;
+ return this;
+ }
+
+ public Builder timestamp(String timestamp) {
+ this.timeStamp = timestamp;
+ return this;
+ }
+
+ public Builder hostname(String hostName) {
+ this.hostName = hostName;
+ return this;
+ }
+
+ public Builder sender(String sender) {
+ this.sender = sender;
+ return this;
+ }
+
+ public Builder msgBody(String msgBody) {
+ this.msgBody = msgBody;
+ return this;
+ }
+
+ public Builder fullMessage(String fullMessage) {
+ this.fullMessage = fullMessage;
+ return this;
+ }
+
+ public Builder rawMessage(byte[] rawMessage) {
+ this.rawMessage = rawMessage;
+ return this;
+ }
+
+ public Builder valid(boolean valid) {
+ this.valid = valid;
+ return this;
+ }
+
+ public SyslogEvent build() {
+ return new SyslogEvent(this);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SimpleKeyProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SimpleKeyProvider.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SimpleKeyProvider.java
new file mode 100644
index 0000000..76959d7
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SimpleKeyProvider.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.syslog.keyproviders;
+
+import com.github.palindromicity.syslog.KeyProvider;
+import org.apache.nifi.syslog.attributes.Syslog5424Attributes;
+import org.apache.nifi.syslog.attributes.SyslogAttributes;
+
+import java.util.regex.Pattern;
+
+public class SimpleKeyProvider implements KeyProvider {
+ private Pattern pattern;
+
+ public SimpleKeyProvider() {
+ }
+
+ @Override
+ public String getMessage() {
+ return SyslogAttributes.BODY.key();
+ }
+
+ @Override
+ public String getHeaderAppName(){
+ return Syslog5424Attributes.APP_NAME.key();
+ }
+
+ @Override
+ public String getHeaderHostName() {
+ return SyslogAttributes.HOSTNAME.key();
+ }
+
+ @Override
+ public String getHeaderPriority() {
+ return SyslogAttributes.PRIORITY.key();
+ }
+
+ @Override
+ public String getHeaderFacility() {
+ return SyslogAttributes.FACILITY.key();
+ }
+
+ @Override
+ public String getHeaderSeverity() {
+ return SyslogAttributes.SEVERITY.key();
+ }
+
+
+ @Override
+ public String getHeaderProcessId() {
+ return Syslog5424Attributes.PROCID.key();
+ }
+
+ @Override
+ public String getHeaderTimeStamp() {
+ return SyslogAttributes.TIMESTAMP.key();
+ }
+
+ @Override
+ public String getHeaderMessageId() {
+ return Syslog5424Attributes.MESSAGEID.key();
+ }
+
+ @Override
+ public String getHeaderVersion() {
+ return SyslogAttributes.VERSION.key();
+ }
+
+ @Override
+ public String getStructuredBase() {
+ return Syslog5424Attributes.STRUCTURED_BASE.key();
+ }
+
+ @Override
+ public String getStructuredElementIdFormat() {
+ return Syslog5424Attributes.STRUCTURED_ELEMENT_ID_FMT.key();
+ }
+
+ @Override
+ public String getStructuredElementIdParamNameFormat() {
+ return Syslog5424Attributes.STRUCTURED_ELEMENT_ID_PNAME_FMT.key();
+ }
+
+ @Override
+ public Pattern getStructuredElementIdParamNamePattern() {
+ if (pattern == null) {
+ pattern = Pattern.compile(Syslog5424Attributes.STRUCTURED_ELEMENT_ID_PNAME_PATTERN.key());
+ }
+ return pattern;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SyslogPrefixedKeyProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SyslogPrefixedKeyProvider.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SyslogPrefixedKeyProvider.java
new file mode 100644
index 0000000..c23aab5
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/keyproviders/SyslogPrefixedKeyProvider.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.syslog.keyproviders;
+
+import com.github.palindromicity.syslog.KeyProvider;
+import org.apache.nifi.syslog.attributes.Syslog5424Attributes;
+import org.apache.nifi.syslog.attributes.SyslogAttributes;
+
+import java.util.regex.Pattern;
+
+public class SyslogPrefixedKeyProvider implements KeyProvider {
+ private Pattern pattern;
+
+ public SyslogPrefixedKeyProvider() {
+ }
+
+ @Override
+ public String getMessage() {
+ return SyslogAttributes.SYSLOG_BODY.key();
+ }
+
+ @Override
+ public String getHeaderAppName() {
+ return Syslog5424Attributes.SYSLOG_APP_NAME.key();
+ }
+
+ @Override
+ public String getHeaderHostName() {
+ return SyslogAttributes.SYSLOG_HOSTNAME.key();
+ }
+
+ @Override
+ public String getHeaderPriority() {
+ return SyslogAttributes.SYSLOG_PRIORITY.key();
+ }
+
+ @Override
+ public String getHeaderFacility() {
+ return SyslogAttributes.SYSLOG_FACILITY.key();
+ }
+
+ @Override
+ public String getHeaderSeverity() {
+ return SyslogAttributes.SYSLOG_SEVERITY.key();
+ }
+
+
+ @Override
+ public String getHeaderProcessId() {
+ return Syslog5424Attributes.SYSLOG_PROCID.key();
+ }
+
+ @Override
+ public String getHeaderTimeStamp() {
+ return SyslogAttributes.SYSLOG_TIMESTAMP.key();
+ }
+
+ @Override
+ public String getHeaderMessageId() {
+ return Syslog5424Attributes.SYSLOG_MESSAGEID.key();
+ }
+
+ @Override
+ public String getHeaderVersion() {
+ return SyslogAttributes.SYSLOG_VERSION.key();
+ }
+
+ @Override
+ public String getStructuredBase() {
+ return Syslog5424Attributes.SYSLOG_STRUCTURED_BASE.key();
+ }
+
+ @Override
+ public String getStructuredElementIdFormat() {
+ return Syslog5424Attributes.SYSLOG_STRUCTURED_ELEMENT_ID_FMT.key();
+ }
+
+ @Override
+ public String getStructuredElementIdParamNameFormat() {
+ return Syslog5424Attributes.SYSLOG_STRUCTURED_ELEMENT_ID_PNAME_FMT.key();
+ }
+
+ @Override
+ public Pattern getStructuredElementIdParamNamePattern() {
+ if (pattern == null) {
+ pattern = Pattern.compile(Syslog5424Attributes.SYSLOG_STRUCTURED_ELEMENT_ID_PNAME_PATTERN.key());
+ }
+ return pattern;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
new file mode 100644
index 0000000..ed36c9d
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/StrictSyslog5424Parser.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog.parsers;
+
+import com.github.palindromicity.syslog.KeyProvider;
+import com.github.palindromicity.syslog.NilPolicy;
+import com.github.palindromicity.syslog.StructuredDataPolicy;
+import com.github.palindromicity.syslog.SyslogParserBuilder;
+import org.apache.nifi.syslog.events.Syslog5424Event;
+import org.apache.nifi.syslog.keyproviders.SyslogPrefixedKeyProvider;
+import org.apache.nifi.syslog.utils.NifiStructuredDataPolicy;
+import org.apache.nifi.syslog.utils.NilHandlingPolicy;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Parses a Syslog message from a ByteBuffer into a Syslog5424Event instance.
+ * For 5424 we use simple-syslog-5424 since it parsers out structured data.
+ */
+public class StrictSyslog5424Parser {
+ private Charset charset;
+ private com.github.palindromicity.syslog.SyslogParser parser;
+
+ public StrictSyslog5424Parser() {
+ this(StandardCharsets.UTF_8, NilHandlingPolicy.NULL, NifiStructuredDataPolicy.FLATTEN, new SyslogPrefixedKeyProvider());
+ }
+
+ public StrictSyslog5424Parser(final Charset charset, final NilHandlingPolicy nilPolicy,
+ NifiStructuredDataPolicy structuredDataPolicy, KeyProvider keyProvider) {
+ this.charset = charset;
+ parser = new SyslogParserBuilder()
+ .withNilPolicy(NilPolicy.valueOf(nilPolicy.name()))
+ .withStructuredDataPolicy(StructuredDataPolicy.valueOf(structuredDataPolicy.name()))
+ .withKeyProvider(keyProvider)
+ .build();
+ }
+
+ /**
+ * Parses a Syslog5424Event from a {@code ByteBuffer}.
+ *
+ * @param buffer a {@code ByteBuffer} containing a syslog message
+ * @return a Syslog5424Event parsed from the {@code {@code byte array}}
+ */
+ public Syslog5424Event parseEvent(final ByteBuffer buffer) {
+ return parseEvent(buffer, null);
+ }
+
+ /**
+ * Parses a Syslog5424Event from a {@code ByteBuffer}.
+ *
+ * @param buffer a {@code ByteBuffer} containing a syslog message
+ * @param sender the hostname of the syslog server that sent the message
+ * @return a Syslog5424Event parsed from the {@code byte array}
+ */
+ public Syslog5424Event parseEvent(final ByteBuffer buffer, final String sender) {
+ if (buffer == null) {
+ return null;
+ }
+ return parseEvent(bufferToBytes(buffer), sender);
+ }
+
+ /**
+ * Parses a Syslog5424Event from a {@code byte array}.
+ *
+ * @param bytes a {@code byte array} containing a syslog message
+ * @param sender the hostname of the syslog server that sent the message
+ * @return a Syslog5424Event parsed from the {@code byte array}
+ */
+ public Syslog5424Event parseEvent(final byte[] bytes, final String sender) {
+ if (bytes == null || bytes.length == 0) {
+ return null;
+ }
+
+ // remove trailing new line before parsing
+ int length = bytes.length;
+ if (bytes[length - 1] == '\n') {
+ length = length - 1;
+ }
+
+ final String message = new String(bytes, 0, length, charset);
+
+ final Syslog5424Event.Builder builder = new Syslog5424Event.Builder()
+ .valid(false).fullMessage(message).rawMessage(bytes).sender(sender);
+
+ try {
+ parser.parseLine(message, builder::fieldMap);
+ builder.valid(true);
+ } catch (Exception e) {
+ // this is not a valid 5424 message
+ builder.valid(false);
+ builder.exception(e);
+ }
+
+ // either invalid w/original msg, or fully parsed event
+ return builder.build();
+ }
+
+ public String getCharsetName() {
+ return charset == null ? StandardCharsets.UTF_8.name() : charset.name();
+ }
+
+
+ private byte[] bufferToBytes(ByteBuffer buffer) {
+ if (buffer == null) {
+ return null;
+ }
+ if (buffer.position() != 0) {
+ buffer.flip();
+ }
+ byte bytes[] = new byte[buffer.limit()];
+ buffer.get(bytes, 0, buffer.limit());
+ return bytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/SyslogParser.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/SyslogParser.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/SyslogParser.java
new file mode 100644
index 0000000..165dc39
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/parsers/SyslogParser.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog.parsers;
+
+import org.apache.nifi.syslog.events.SyslogEvent;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.MatchResult;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Parses a Syslog message from a ByteBuffer into a SyslogEvent instance.
+ *
+ * The Syslog regular expressions below were adapted from the Apache Flume project.
+ */
+public class SyslogParser {
+
+ public static final String SYSLOG_MSG_RFC5424_0 =
+ "(?:\\<(\\d{1,3})\\>)" + // priority
+ "(?:(\\d)?\\s?)" + // version
+ /* yyyy-MM-dd'T'HH:mm:ss.SZ or yyyy-MM-dd'T'HH:mm:ss.S+hh:mm or - (null stamp) */
+ "(?:" +
+ "(\\d{4}[-]\\d{2}[-]\\d{2}[T]\\d{2}[:]\\d{2}[:]\\d{2}" +
+ "(?:\\.\\d{1,6})?(?:[+-]\\d{2}[:]\\d{2}|Z)?)|-)" + // stamp
+ "\\s" + // separator
+ "(?:([\\w][\\w\\d\\.@\\-]*)|-)" + // host name or - (null)
+ "\\s" + // separator
+ "(.*)$"; // body
+
+ public static final String SYSLOG_MSG_RFC3164_0 =
+ "(?:\\<(\\d{1,3})\\>)" +
+ "(?:(\\d)?\\s?)" + // version
+ // stamp MMM d HH:mm:ss, single digit date has two spaces
+ "([A-Z][a-z][a-z]\\s{1,2}\\d{1,2}\\s\\d{2}[:]\\d{2}[:]\\d{2})" +
+ "\\s" + // separator
+ "([\\w][\\w\\d(\\.|\\:)@-]*)" + // host
+ "\\s(.*)$"; // body
+
+ public static final Collection<Pattern> MESSAGE_PATTERNS;
+ static {
+ List<Pattern> patterns = new ArrayList<>();
+ patterns.add(Pattern.compile(SYSLOG_MSG_RFC5424_0));
+ patterns.add(Pattern.compile(SYSLOG_MSG_RFC3164_0));
+ MESSAGE_PATTERNS = Collections.unmodifiableList(patterns);
+ }
+
+ // capture group positions from the above message patterns
+ public static final int SYSLOG_PRIORITY_POS = 1;
+ public static final int SYSLOG_VERSION_POS = 2;
+ public static final int SYSLOG_TIMESTAMP_POS = 3;
+ public static final int SYSLOG_HOSTNAME_POS = 4;
+ public static final int SYSLOG_BODY_POS = 5;
+
+ private Charset charset;
+
+ public SyslogParser() {
+ this(StandardCharsets.UTF_8);
+ }
+
+ public SyslogParser(final Charset charset) {
+ this.charset = charset;
+ }
+
+ /**
+ * Parses a SyslogEvent from a byte buffer.
+ *
+ * @param buffer a byte buffer containing a syslog message
+ * @return a SyslogEvent parsed from the byte array
+ */
+ public SyslogEvent parseEvent(final ByteBuffer buffer) {
+ return parseEvent(buffer, null);
+ }
+
+ /**
+ * Parses a SyslogEvent from a byte buffer.
+ *
+ * @param buffer a byte buffer containing a syslog message
+ * @param sender the hostname of the syslog server that sent the message
+ * @return a SyslogEvent parsed from the byte array
+ */
+ public SyslogEvent parseEvent(final ByteBuffer buffer, final String sender) {
+ if (buffer == null) {
+ return null;
+ }
+ if (buffer.position() != 0) {
+ buffer.flip();
+ }
+ byte bytes[] = new byte[buffer.limit()];
+ buffer.get(bytes, 0, buffer.limit());
+ return parseEvent(bytes, sender);
+ }
+
+ /**
+ * Parses a SyslogEvent from a byte array.
+ *
+ * @param bytes a byte array containing a syslog message
+ * @param sender the hostname of the syslog server that sent the message
+ * @return a SyslogEvent parsed from the byte array
+ */
+ public SyslogEvent parseEvent(final byte[] bytes, final String sender) {
+ if (bytes == null || bytes.length == 0) {
+ return null;
+ }
+
+ // remove trailing new line before parsing
+ int length = bytes.length;
+ if (bytes[length - 1] == '\n') {
+ length = length - 1;
+ }
+
+ final String message = new String(bytes, 0, length, charset);
+
+ final SyslogEvent.Builder builder = new SyslogEvent.Builder()
+ .valid(false).fullMessage(message).rawMessage(bytes).sender(sender);
+
+ for (Pattern pattern : MESSAGE_PATTERNS) {
+ final Matcher matcher = pattern.matcher(message);
+ if (!matcher.matches()) {
+ continue;
+ }
+
+ final MatchResult res = matcher.toMatchResult();
+ for (int grp = 1; grp <= res.groupCount(); grp++) {
+ String value = res.group(grp);
+ if (grp == SYSLOG_TIMESTAMP_POS) {
+ builder.timestamp(value);
+ } else if (grp == SYSLOG_HOSTNAME_POS) {
+ builder.hostname(value);
+ } else if (grp == SYSLOG_PRIORITY_POS) {
+ int pri = Integer.parseInt(value);
+ int sev = pri % 8;
+ int facility = pri / 8;
+ builder.priority(value);
+ builder.severity(String.valueOf(sev));
+ builder.facility(String.valueOf(facility));
+ } else if (grp == SYSLOG_VERSION_POS) {
+ builder.version(value);
+ } else if (grp == SYSLOG_BODY_POS) {
+ builder.msgBody(value);
+ }
+ }
+
+ builder.valid(true);
+ break;
+ }
+
+ // either invalid w/original msg, or fully parsed event
+ return builder.build();
+ }
+
+ public String getCharsetName() {
+ return charset == null ? StandardCharsets.UTF_8.name() : charset.name();
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NifiStructuredDataPolicy.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NifiStructuredDataPolicy.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NifiStructuredDataPolicy.java
new file mode 100644
index 0000000..5af2191
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NifiStructuredDataPolicy.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.syslog.utils;
+
+/**
+ * Policy for handling Structured Data
+ * must match the simple-syslog-5424 StructuredDataPolicy
+ */
+public enum NifiStructuredDataPolicy {
+ /**
+ * The Structured Data will be flattened per the KeyProvider provided values.
+ */
+ FLATTEN,
+ /**
+ * The Structued Data will be returned as a Map field named structuredData.
+ * Each map entry will have the value of the Structured Data ID, and a value
+ * of a map of each element param name and value
+ */
+ MAP_OF_MAPS
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NilHandlingPolicy.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NilHandlingPolicy.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NilHandlingPolicy.java
new file mode 100644
index 0000000..d2c3d3c
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/main/java/org/apache/nifi/syslog/utils/NilHandlingPolicy.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog.utils;
+
+/**
+ * Policies available for handling NIL '-' values.
+ * Must match the simple-syslog-5424 NilPolicy
+ */
+public enum NilHandlingPolicy {
+ /**
+ * a nil value will result msg part being omitted from the map.
+ */
+ OMIT,
+ /**
+ * a nil value will result in a null value in the map.
+ */
+ NULL,
+ /**
+ * a nil value will result in a '-' symbol in the map.
+ */
+ DASH
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/BaseStrictSyslog5424ParserTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/BaseStrictSyslog5424ParserTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/BaseStrictSyslog5424ParserTest.java
new file mode 100644
index 0000000..082fbbd
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/BaseStrictSyslog5424ParserTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog;
+
+import org.apache.nifi.syslog.attributes.Syslog5424Attributes;
+import org.apache.nifi.syslog.attributes.SyslogAttributes;
+import org.apache.nifi.syslog.events.Syslog5424Event;
+import org.apache.nifi.syslog.keyproviders.SyslogPrefixedKeyProvider;
+import org.apache.nifi.syslog.parsers.StrictSyslog5424Parser;
+import org.apache.nifi.syslog.utils.NifiStructuredDataPolicy;
+import org.apache.nifi.syslog.utils.NilHandlingPolicy;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+public abstract class BaseStrictSyslog5424ParserTest {
+
+ private static final Charset CHARSET = Charset.forName("UTF-8");
+ private static final String NIL_VALUE = "-";
+ private StrictSyslog5424Parser parser;
+
+ protected abstract NilHandlingPolicy getPolicy();
+
+ protected void validateForPolicy(String expected, Object actual) {
+ switch (getPolicy()) {
+ case DASH:
+ Assert.assertEquals(actual, NIL_VALUE);
+ break;
+ case OMIT:
+ case NULL:
+ Assert.assertNull(actual);
+
+ }
+ }
+
+ @Before
+ public void setup() {
+ parser = new StrictSyslog5424Parser(CHARSET, getPolicy(), NifiStructuredDataPolicy.FLATTEN, new SyslogPrefixedKeyProvider());
+ }
+
+ @Test
+ public void testRFC5424WithVersion() {
+ final String pri = "34";
+ final String version = "1";
+ final String stamp = "2003-10-11T22:14:15.003Z";
+ final String host = "mymachine.example.com";
+ final String appName = "su";
+ final String procId = "-";
+ final String msgId = "ID17";
+ final String structuredData = "-";
+ final String body = "BOM'su root' failed for lonvick on /dev/pts/8";
+
+ final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " "
+ + appName + " " + procId + " " + msgId + " " + "-" + " " + body;
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final Syslog5424Event event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertTrue(event.isValid());
+ Assert.assertFalse(event.getFieldMap().isEmpty());
+ Map<String, Object> fieldMap = event.getFieldMap();
+ Assert.assertEquals(pri, fieldMap.get(SyslogAttributes.SYSLOG_PRIORITY.key()));
+ Assert.assertEquals("2", fieldMap.get(SyslogAttributes.SYSLOG_SEVERITY.key()));
+ Assert.assertEquals("4", fieldMap.get(SyslogAttributes.SYSLOG_FACILITY.key()));
+ Assert.assertEquals(version, fieldMap.get(SyslogAttributes.SYSLOG_VERSION.key()));
+ Assert.assertEquals(stamp, fieldMap.get(SyslogAttributes.SYSLOG_TIMESTAMP.key()));
+ Assert.assertEquals(host, fieldMap.get(SyslogAttributes.SYSLOG_HOSTNAME.key()));
+ Assert.assertEquals(appName, fieldMap.get(Syslog5424Attributes.SYSLOG_APP_NAME.key()));
+ validateForPolicy(procId, fieldMap.get(Syslog5424Attributes.SYSLOG_PROCID.key()));
+ Assert.assertEquals(msgId, fieldMap.get(Syslog5424Attributes.SYSLOG_MESSAGEID.key()));
+
+ Pattern structuredPattern = new SyslogPrefixedKeyProvider().getStructuredElementIdParamNamePattern();
+ fieldMap.forEach((key, value) -> {
+ if (value != null) {
+ Assert.assertFalse(structuredPattern.matcher(key).matches());
+ }
+ });
+
+ Assert.assertEquals(body, fieldMap.get(SyslogAttributes.SYSLOG_BODY.key()));
+ Assert.assertEquals(message, event.getFullMessage());
+ Assert.assertNull(event.getSender());
+ }
+
+ @Test
+ public void testRFC5424WithoutVersion() {
+ final String pri = "34";
+ final String version = "-";
+ final String stamp = "2003-10-11T22:14:15.003Z";
+ final String host = "mymachine.example.com";
+ final String appName = "su";
+ final String procId = "-";
+ final String msgId = "ID17";
+ final String structuredData = "-";
+ final String body = "BOM'su root' failed for lonvick on /dev/pts/8";
+
+ final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " "
+ + appName + " " + procId + " " + msgId + " " + "-" + " " + body;
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final Syslog5424Event event = parser.parseEvent(buffer);
+ Assert.assertFalse(event.isValid());
+ }
+
+ @Test
+ public void testTrailingNewLine() {
+ final String message = "<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - " +
+ "ID47 - BOM'su root' failed for lonvick on /dev/pts/8\n";
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final Syslog5424Event event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertTrue(event.isValid());
+ }
+
+ @Test
+ public void testVariety() {
+ final List<String> messages = new ArrayList<>();
+
+ // supported examples from RFC 5424
+ messages.add("<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - " +
+ "ID47 - BOM'su root' failed for lonvick on /dev/pts/8");
+ messages.add("<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc " +
+ "8710 - - %% It's time to make the do-nuts.");
+ messages.add("<14>1 2014-06-20T09:14:07+00:00 loggregator"
+ + " d0602076-b14a-4c55-852a-981e7afeed38 DEA MSG-01"
+ + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+ + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance");
+
+ for (final String message : messages) {
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final Syslog5424Event event = parser.parseEvent(buffer);
+ Assert.assertTrue(event.isValid());
+ }
+ }
+
+ @Test
+ public void testInvalidPriority() {
+ final String message = "10 Oct 13 14:14:43 localhost some body of the message";
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final Syslog5424Event event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertFalse(event.isValid());
+ Assert.assertEquals(message, event.getFullMessage());
+ }
+
+ @Test
+ public void testParseWithSender() {
+ final String sender = "127.0.0.1";
+ final String message = "<14>1 2014-06-20T09:14:07+00:00 loggregator"
+ + " d0602076-b14a-4c55-852a-981e7afeed38 DEA MSG-01"
+ + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+ + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance";
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final Syslog5424Event event = parser.parseEvent(buffer, sender);
+ Assert.assertNotNull(event);
+ Assert.assertTrue(event.isValid());
+ Assert.assertEquals(sender, event.getSender());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserDashPolicyTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserDashPolicyTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserDashPolicyTest.java
new file mode 100644
index 0000000..c330551
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserDashPolicyTest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog;
+
+
+import org.apache.nifi.syslog.utils.NilHandlingPolicy;
+
+public class StrictSyslog5424ParserDashPolicyTest extends BaseStrictSyslog5424ParserTest {
+ protected NilHandlingPolicy getPolicy() {
+ return NilHandlingPolicy.DASH;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserNullPolicyTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserNullPolicyTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserNullPolicyTest.java
new file mode 100644
index 0000000..921d9bf
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserNullPolicyTest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog;
+
+
+import org.apache.nifi.syslog.utils.NilHandlingPolicy;
+
+public class StrictSyslog5424ParserNullPolicyTest extends BaseStrictSyslog5424ParserTest {
+ protected NilHandlingPolicy getPolicy() {
+ return NilHandlingPolicy.NULL;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserOmitPolicyTest.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserOmitPolicyTest.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserOmitPolicyTest.java
new file mode 100644
index 0000000..04bc151
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/StrictSyslog5424ParserOmitPolicyTest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog;
+
+
+import org.apache.nifi.syslog.utils.NilHandlingPolicy;
+
+public class StrictSyslog5424ParserOmitPolicyTest extends BaseStrictSyslog5424ParserTest {
+ protected NilHandlingPolicy getPolicy() {
+ return NilHandlingPolicy.OMIT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/TestSyslogParser.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/TestSyslogParser.java b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/TestSyslogParser.java
new file mode 100644
index 0000000..054d1eb
--- /dev/null
+++ b/nifi-nar-bundles/nifi-extension-utils/nifi-syslog-utils/src/test/java/org/apache/nifi/syslog/TestSyslogParser.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.syslog;
+
+import org.apache.nifi.syslog.events.SyslogEvent;
+import org.apache.nifi.syslog.parsers.SyslogParser;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TestSyslogParser {
+
+ static final Charset CHARSET = Charset.forName("UTF-8");
+
+ private SyslogParser parser;
+
+ @Before
+ public void setup() {
+ parser = new SyslogParser(CHARSET);
+ }
+
+ @Test
+ public void testRFC3164SingleDigitDay() {
+ final String pri = "10";
+ final String stamp = "Oct 1 13:14:04";
+ final String host = "my.host.com";
+ final String body = "some body message";
+ final String message = "<" + pri + ">" + stamp + " " + host + " " + body;
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertEquals(pri, event.getPriority());
+ Assert.assertEquals("2", event.getSeverity());
+ Assert.assertEquals("1", event.getFacility());
+ Assert.assertNull(event.getVersion());
+ Assert.assertEquals(stamp, event.getTimeStamp());
+ Assert.assertEquals(host, event.getHostName());
+ Assert.assertEquals(body, event.getMsgBody());
+ Assert.assertEquals(message, event.getFullMessage());
+ Assert.assertTrue(event.isValid());
+ }
+
+ @Test
+ public void testRFC3164DoubleDigitDay() {
+ final String pri = "31";
+ final String stamp = "Oct 13 14:14:43";
+ final String host = "localhost";
+ final String body = "AppleCameraAssistant[470]: DeviceMessageNotificationCallback: kIOPMMessageSystemPowerEventOccurred: 0x00000000";
+ final String message = "<" + pri + ">" + stamp + " " + host + " " + body;
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertEquals(pri, event.getPriority());
+ Assert.assertEquals("7", event.getSeverity());
+ Assert.assertEquals("3", event.getFacility());
+ Assert.assertNull(event.getVersion());
+ Assert.assertEquals(stamp, event.getTimeStamp());
+ Assert.assertEquals(host, event.getHostName());
+ Assert.assertEquals(body, event.getMsgBody());
+ Assert.assertEquals(message, event.getFullMessage());
+ Assert.assertTrue(event.isValid());
+ }
+
+ @Test
+ public void testRFC3164WithVersion() {
+ final String pri = "31";
+ final String version = "1";
+ final String stamp = "Oct 13 14:14:43";
+ final String host = "localhost";
+ final String body = "AppleCameraAssistant[470]: DeviceMessageNotificationCallback: kIOPMMessageSystemPowerEventOccurred: 0x00000000";
+ final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " " + body;
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertEquals(pri, event.getPriority());
+ Assert.assertEquals("7", event.getSeverity());
+ Assert.assertEquals("3", event.getFacility());
+ Assert.assertEquals(version, event.getVersion());
+ Assert.assertEquals(stamp, event.getTimeStamp());
+ Assert.assertEquals(host, event.getHostName());
+ Assert.assertEquals(body, event.getMsgBody());
+ Assert.assertEquals(message, event.getFullMessage());
+ Assert.assertTrue(event.isValid());
+ }
+
+ @Test
+ public void testRFC5424WithVersion() {
+ final String pri = "34";
+ final String version = "1";
+ final String stamp = "2003-10-11T22:14:15.003Z";
+ final String host = "mymachine.example.com";
+ final String body = "su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8";
+
+ final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " " + body;
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertEquals(pri, event.getPriority());
+ Assert.assertEquals("2", event.getSeverity());
+ Assert.assertEquals("4", event.getFacility());
+ Assert.assertEquals(version, event.getVersion());
+ Assert.assertEquals(stamp, event.getTimeStamp());
+ Assert.assertEquals(host, event.getHostName());
+ Assert.assertEquals(body, event.getMsgBody());
+ Assert.assertEquals(message, event.getFullMessage());
+ Assert.assertTrue(event.isValid());
+ }
+
+ @Test
+ public void testRFC5424WithoutVersion() {
+ final String pri = "34";
+ final String stamp = "2003-10-11T22:14:15.003Z";
+ final String host = "mymachine.example.com";
+ final String body = "su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8";
+
+ final String message = "<" + pri + ">" + stamp + " " + host + " " + body;
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertEquals(pri, event.getPriority());
+ Assert.assertEquals("2", event.getSeverity());
+ Assert.assertEquals("4", event.getFacility());
+ Assert.assertNull(event.getVersion());
+ Assert.assertEquals(stamp, event.getTimeStamp());
+ Assert.assertEquals(host, event.getHostName());
+ Assert.assertEquals(body, event.getMsgBody());
+ Assert.assertEquals(message, event.getFullMessage());
+ Assert.assertTrue(event.isValid());
+ }
+
+ @Test
+ public void testTrailingNewLine() {
+ final String message = "<31>Oct 13 15:43:23 localhost.home some message\n";
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertTrue(event.isValid());
+ }
+
+ @Test
+ public void testVariety() {
+ final List<String> messages = new ArrayList<>();
+
+ // supported examples from RFC 3164
+ messages.add("<34>Oct 11 22:14:15 mymachine su: 'su root' failed for " +
+ "lonvick on /dev/pts/8");
+ messages.add("<13>Feb 5 17:32:18 10.0.0.99 Use the BFG!");
+ messages.add("<165>Aug 24 05:34:00 CST 1987 mymachine myproc[10]: %% " +
+ "It's time to make the do-nuts. %% Ingredients: Mix=OK, Jelly=OK # " +
+ "Devices: Mixer=OK, Jelly_Injector=OK, Frier=OK # Transport: " +
+ "Conveyer1=OK, Conveyer2=OK # %%");
+ messages.add("<0>Oct 22 10:52:12 scapegoat 1990 Oct 22 10:52:01 TZ-6 " +
+ "scapegoat.dmz.example.org 10.1.2.3 sched[0]: That's All Folks!");
+
+ // supported examples from RFC 5424
+ messages.add("<34>1 2003-10-11T22:14:15.003Z mymachine.example.com su - " +
+ "ID47 - BOM'su root' failed for lonvick on /dev/pts/8");
+ messages.add("<165>1 2003-08-24T05:14:15.000003-07:00 192.0.2.1 myproc " +
+ "8710 - - %% It's time to make the do-nuts.");
+
+ // non-standard (but common) messages (RFC3339 dates, no version digit)
+ messages.add("<13>2003-08-24T05:14:15Z localhost snarf?");
+ messages.add("<13>2012-08-16T14:34:03-08:00 127.0.0.1 test shnap!");
+
+ for (final String message : messages) {
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer);
+ Assert.assertTrue(event.isValid());
+ }
+ }
+
+ @Test
+ public void testInvalidPriority() {
+ final String message = "10 Oct 13 14:14:43 localhost some body of the message";
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer);
+ Assert.assertNotNull(event);
+ Assert.assertFalse(event.isValid());
+ Assert.assertEquals(message, event.getFullMessage());
+ }
+
+ @Test
+ public void testParseWithSender() {
+ final String sender = "127.0.0.1";
+ final String message = "<31>Oct 13 15:43:23 localhost.home some message\n";
+
+ final byte[] bytes = message.getBytes(CHARSET);
+ final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
+ buffer.clear();
+ buffer.put(bytes);
+
+ final SyslogEvent event = parser.parseEvent(buffer, sender);
+ Assert.assertNotNull(event);
+ Assert.assertTrue(event.isValid());
+ Assert.assertEquals(sender, event.getSender());
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-extension-utils/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-extension-utils/pom.xml b/nifi-nar-bundles/nifi-extension-utils/pom.xml
index 289cb55..85f2a4e 100644
--- a/nifi-nar-bundles/nifi-extension-utils/pom.xml
+++ b/nifi-nar-bundles/nifi-extension-utils/pom.xml
@@ -31,6 +31,7 @@
<module>nifi-hadoop-utils</module>
<module>nifi-processor-utils</module>
<module>nifi-reporting-utils</module>
+ <module>nifi-syslog-utils</module>
</modules>
</project>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index a3fe1f6..39d7a6d 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -31,6 +31,11 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-syslog-utils</artifactId>
+ <version>1.8.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-utils</artifactId>
<version>1.8.0-SNAPSHOT</version>
</dependency>
@@ -327,10 +332,6 @@
<version>1.8.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>com.github.palindromicity</groupId>
- <artifactId>simple-syslog-5424</artifactId>
- </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java
index fb239ee..0509e0c 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListenSyslog.java
@@ -48,12 +48,12 @@ import org.apache.nifi.processor.util.listen.event.EventFactory;
import org.apache.nifi.processor.util.listen.handler.ChannelHandlerFactory;
import org.apache.nifi.processor.util.listen.handler.socket.SocketChannelHandlerFactory;
import org.apache.nifi.processor.util.listen.response.ChannelResponder;
-import org.apache.nifi.processors.standard.syslog.SyslogAttributes;
-import org.apache.nifi.processors.standard.syslog.SyslogEvent;
-import org.apache.nifi.processors.standard.syslog.SyslogParser;
import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.ssl.RestrictedSSLContextService;
import org.apache.nifi.ssl.SSLContextService;
+import org.apache.nifi.syslog.attributes.SyslogAttributes;
+import org.apache.nifi.syslog.events.SyslogEvent;
+import org.apache.nifi.syslog.parsers.SyslogParser;
import javax.net.ssl.SSLContext;
import java.io.IOException;
@@ -419,8 +419,8 @@ public class ListenSyslog extends AbstractSyslogProcessor {
final String protocol = context.getProperty(PROTOCOL).getValue();
final Map<String, String> defaultAttributes = new HashMap<>(4);
- defaultAttributes.put(SyslogAttributes.PROTOCOL.key(), protocol);
- defaultAttributes.put(SyslogAttributes.PORT.key(), port);
+ defaultAttributes.put(SyslogAttributes.SYSLOG_PROTOCOL.key(), protocol);
+ defaultAttributes.put(SyslogAttributes.SYSLOG_PORT.key(), port);
defaultAttributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain");
@@ -461,7 +461,7 @@ public class ListenSyslog extends AbstractSyslogProcessor {
FlowFile invalidFlowFile = session.create();
invalidFlowFile = session.putAllAttributes(invalidFlowFile, defaultAttributes);
if (sender != null) {
- invalidFlowFile = session.putAttribute(invalidFlowFile, SyslogAttributes.SENDER.key(), sender);
+ invalidFlowFile = session.putAttribute(invalidFlowFile, SyslogAttributes.SYSLOG_SENDER.key(), sender);
}
try {
@@ -486,14 +486,14 @@ public class ListenSyslog extends AbstractSyslogProcessor {
getLogger().trace(event.getFullMessage());
final Map<String, String> attributes = new HashMap<>(numAttributes);
- attributes.put(SyslogAttributes.PRIORITY.key(), event.getPriority());
- attributes.put(SyslogAttributes.SEVERITY.key(), event.getSeverity());
- attributes.put(SyslogAttributes.FACILITY.key(), event.getFacility());
- attributes.put(SyslogAttributes.VERSION.key(), event.getVersion());
- attributes.put(SyslogAttributes.TIMESTAMP.key(), event.getTimeStamp());
- attributes.put(SyslogAttributes.HOSTNAME.key(), event.getHostName());
- attributes.put(SyslogAttributes.BODY.key(), event.getMsgBody());
- attributes.put(SyslogAttributes.VALID.key(), String.valueOf(event.isValid()));
+ attributes.put(SyslogAttributes.SYSLOG_PRIORITY.key(), event.getPriority());
+ attributes.put(SyslogAttributes.SYSLOG_SEVERITY.key(), event.getSeverity());
+ attributes.put(SyslogAttributes.SYSLOG_FACILITY.key(), event.getFacility());
+ attributes.put(SyslogAttributes.SYSLOG_VERSION.key(), event.getVersion());
+ attributes.put(SyslogAttributes.SYSLOG_TIMESTAMP.key(), event.getTimeStamp());
+ attributes.put(SyslogAttributes.SYSLOG_HOSTNAME.key(), event.getHostName());
+ attributes.put(SyslogAttributes.SYSLOG_BODY.key(), event.getMsgBody());
+ attributes.put(SyslogAttributes.SYSLOG_VALID.key(), String.valueOf(event.isValid()));
flowFile = session.putAllAttributes(flowFile, attributes);
}
@@ -536,7 +536,7 @@ public class ListenSyslog extends AbstractSyslogProcessor {
final Map<String, String> newAttributes = new HashMap<>(defaultAttributes.size() + 1);
newAttributes.putAll(defaultAttributes);
- newAttributes.put(SyslogAttributes.SENDER.key(), sender);
+ newAttributes.put(SyslogAttributes.SYSLOG_SENDER.key(), sender);
flowFile = session.putAllAttributes(flowFile, newAttributes);
getLogger().debug("Transferring {} to success", new Object[] {flowFile});
http://git-wip-us.apache.org/repos/asf/nifi/blob/b1022043/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java
index 31691eb..d8f8b8b 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ParseSyslog.java
@@ -17,15 +17,6 @@
package org.apache.nifi.processors.standard;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
@@ -45,10 +36,20 @@ import org.apache.nifi.processor.Relationship;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.io.InputStreamCallback;
import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.processors.standard.syslog.SyslogAttributes;
-import org.apache.nifi.processors.standard.syslog.SyslogEvent;
-import org.apache.nifi.processors.standard.syslog.SyslogParser;
import org.apache.nifi.stream.io.StreamUtils;
+import org.apache.nifi.syslog.attributes.SyslogAttributes;
+import org.apache.nifi.syslog.events.SyslogEvent;
+import org.apache.nifi.syslog.parsers.SyslogParser;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
@EventDriven
@@ -145,13 +146,13 @@ public class ParseSyslog extends AbstractProcessor {
}
final Map<String, String> attributes = new HashMap<>(8);
- attributes.put(SyslogAttributes.PRIORITY.key(), event.getPriority());
- attributes.put(SyslogAttributes.SEVERITY.key(), event.getSeverity());
- attributes.put(SyslogAttributes.FACILITY.key(), event.getFacility());
- attributes.put(SyslogAttributes.VERSION.key(), event.getVersion());
- attributes.put(SyslogAttributes.TIMESTAMP.key(), event.getTimeStamp());
- attributes.put(SyslogAttributes.HOSTNAME.key(), event.getHostName());
- attributes.put(SyslogAttributes.BODY.key(), event.getMsgBody());
+ attributes.put(SyslogAttributes.SYSLOG_PRIORITY.key(), event.getPriority());
+ attributes.put(SyslogAttributes.SYSLOG_SEVERITY.key(), event.getSeverity());
+ attributes.put(SyslogAttributes.SYSLOG_FACILITY.key(), event.getFacility());
+ attributes.put(SyslogAttributes.SYSLOG_VERSION.key(), event.getVersion());
+ attributes.put(SyslogAttributes.SYSLOG_TIMESTAMP.key(), event.getTimeStamp());
+ attributes.put(SyslogAttributes.SYSLOG_HOSTNAME.key(), event.getHostName());
+ attributes.put(SyslogAttributes.SYSLOG_BODY.key(), event.getMsgBody());
flowFile = session.putAllAttributes(flowFile, attributes);
session.transfer(flowFile, REL_SUCCESS);