You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ot...@apache.org on 2018/10/24 15:04:09 UTC
[20/51] [abbrv] metron git commit: METRON-1750 Create Parser for
Syslog RFC 5424 Messages (ottobackwards) closes apache/metron#1175
METRON-1750 Create Parser for Syslog RFC 5424 Messages (ottobackwards) closes apache/metron#1175
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ff1f9cf5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ff1f9cf5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ff1f9cf5
Branch: refs/heads/feature/METRON-1090-stellar-assignment
Commit: ff1f9cf52b31ae866eb3de1cf8993af97af1790f
Parents: c0fb262
Author: ottobackwards <ot...@gmail.com>
Authored: Mon Oct 1 10:23:38 2018 -0400
Committer: otto <ot...@apache.org>
Committed: Mon Oct 1 10:23:38 2018 -0400
----------------------------------------------------------------------
dependencies_with_url.csv | 2 +-
.../docker/rpm-docker/SPECS/metron.spec | 3 +
.../data/syslog5424/parsed/Syslog5424Parsed | 3 +
.../sample/data/syslog5424/raw/Syslog5424Output | 3 +
metron-platform/metron-parsers/README.md | 5 +-
metron-platform/metron-parsers/pom.xml | 5 +
.../config/zookeeper/parsers/syslog5424.json | 7 +
.../metron/parsers/syslog/Syslog5424Parser.java | 102 +++++++++++++
.../Syslog5424ParserIntegrationTest.java | 37 +++++
.../parsers/syslog/Syslog5424ParserTest.java | 146 +++++++++++++++++++
pom.xml | 1 +
11 files changed, 311 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index ffd0fbc..53977f3 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -488,4 +488,4 @@ com.google.code.gson:gson:jar:2.8.2:compile,ASLv2,https://github.com/google/gson
org.sonatype.sisu:sisu-inject-plexus:jar:2.2.2:compile
com.zaxxer:HikariCP:jar:2.7.8:compile,ASLv2,https://github.com/brettwooldridge/HikariCP
org.hibernate.validator:hibernate-validator:jar:6.0.9.Final:compile,ASLv2,https://github.com/hibernate/hibernate-validator
-
+com.github.palindromicity:simple-syslog-5424:jar:0.0.8:compile,ASLv2,https://github.com/palindromicity/simple-syslog-5424
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 4bcef33..ed22a28 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -158,6 +158,7 @@ This package installs the Metron Parser files
%{metron_home}/config/zookeeper/parsers/jsonMapWrappedQuery.json
%{metron_home}/config/zookeeper/parsers/snort.json
%{metron_home}/config/zookeeper/parsers/squid.json
+%{metron_home}/config/zookeeper/parsers/syslog5424.json
%{metron_home}/config/zookeeper/parsers/websphere.json
%{metron_home}/config/zookeeper/parsers/yaf.json
%{metron_home}/config/zookeeper/parsers/asa.json
@@ -590,6 +591,8 @@ chkconfig --del metron-alerts-ui
%changelog
* Thu Aug 30 2018 Apache Metron <de...@metron.apache.org> - 0.6.1
- Update compiled css file name for Alerts UI
+* Fri Aug 24 2018 Apache Metron <de...@metron.apache.org> - 0.6.1
+- Add syslog5424 parser
* Tue Aug 21 2018 Apache Metron <de...@metron.apache.org> - 0.6.1
- Add Profiler for REPL
* Tue Aug 14 2018 Apache Metron <de...@metron.apache.org> - 0.5.1
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
new file mode 100644
index 0000000..e330204
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/parsed/Syslog5424Parsed
@@ -0,0 +1,3 @@
+{"syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed38","syslog.header.version":"1","syslog.header.hostName":"loggregator","original_string":"<14>1 2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA - - Removing instance","syslog.header.facility":"1","syslog.header.msgId":"-","syslog.header.timestamp":"2014-06-20T09:14:07+00:00","syslog.message":"Removing instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","timestamp":"2014-06-20T09:14:07+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
+{"syslog.structuredData.exampleSDID@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other Application","syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.structuredData.exampleSDID@32480.eventID":"2022","syslog.structuredData.exampleSDID@32473.eventSource":"Application","syslog.header.timestamp":"2014-06-20T09:14:08+00:00","syslog.message":"Removing instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.header.severity":"6","syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed40","syslog.header.version":"1","syslog.structuredData.exampleSDID@32473.iut":"3","original_string":"<14>1 2014-06-20T09:14:08+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed40 DEA MSG-02 [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance","syslog.header.msgId":"MSG-02","syslog.structuredData.exampleSDID@32473.eventI
D":"1011","timestamp":"2014-06-20T09:14:08+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
+{"syslog.structuredData.exampleSDID@32480.iut":"4","syslog.structuredData.exampleSDID@32480.eventSource":"Other Application","syslog.structuredData.exampleSDID@32474.iut":"3","syslog.structuredData.exampleSDID@32474.eventID":"1011","syslog.header.hostName":"loggregator","syslog.header.facility":"1","syslog.structuredData.exampleSDID@32480.eventID":"2022","syslog.header.timestamp":"2014-06-20T09:14:09+00:00","syslog.message":"Removing instance","syslog.header.pri":"14","syslog.header.procId":"DEA","syslog.structuredData.exampleSDID@32474.eventSource":"Application","syslog.header.severity":"6","syslog.header.appName":"d0602076-b14a-4c55-852a-981e7afeed42","syslog.header.version":"1","original_string":"<14>1 2014-06-20T09:14:09+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed42 DEA MSG-03 [exampleSDID@32474 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"] [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance","syslog.header.msgId"
:"MSG-03","timestamp":"2014-06-20T09:14:09+00:00","guid":"this-is-random-uuid-will-be-36-chars","source.type":"syslog5424"}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
new file mode 100644
index 0000000..869797a
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/syslog5424/raw/Syslog5424Output
@@ -0,0 +1,3 @@
+<14>1 2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA - - Removing instance
+<14>1 2014-06-20T09:14:08+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed40 DEA MSG-02 [exampleSDID@32473 iut="3" eventSource="Application" eventID="1011"] [exampleSDID@32480 iut="4" eventSource="Other Application" eventID="2022"] Removing instance
+<14>1 2014-06-20T09:14:09+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed42 DEA MSG-03 [exampleSDID@32474 iut="3" eventSource="Application" eventID="1011"] [exampleSDID@32480 iut="4" eventSource="Other Application" eventID="2022"] Removing instance
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/README.md b/metron-platform/metron-parsers/README.md
index fd6b470..381b0ee 100644
--- a/metron-platform/metron-parsers/README.md
+++ b/metron-platform/metron-parsers/README.md
@@ -24,7 +24,7 @@ Parsers are pluggable components which are used to transform raw data
enrichment and indexing.
There are two general types types of parsers:
-* A parser written in Java which conforms to the `MessageParser` interface. This kind of parser is optimized for speed and performance and is built for use with higher velocity topologies. These parsers are not easily modifiable and in order to make changes to them the entire topology need to be recompiled.
+* A parser written in Java which conforms to the `MessageParser` interface. This kind of parser is optimized for speed and performance and is built for use with higher velocity topologies. These parsers are not easily modifiable and in order to make changes to them the entire topology need to be recompiled.
* A general purpose parser. This type of parser is primarily designed for lower-velocity topologies or for quickly standing up a parser for a new telemetry before a permanent Java parser can be written for it. As of the time of this writing, we have:
* Grok parser: `org.apache.metron.parsers.GrokParser` with possible `parserConfig` entries of
* `grokPath` : The path in HDFS (or in the Jar) to the grok statement
@@ -504,12 +504,13 @@ Parser adapters are loaded dynamically in each Metron topology. They
are defined in the Parser Config (defined above) JSON file in Zookeeper.
### Java Parser Adapters
-Java parser adapters are indended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:
+Java parser adapters are intended for higher-velocity topologies and are not easily changed or extended. As the adoption of Metron continues we plan on extending our library of Java adapters to process more log formats. As of this moment the Java adapters included with Metron are:
* org.apache.metron.parsers.ise.BasicIseParser : Parse ISE messages
* org.apache.metron.parsers.bro.BasicBroParser : Parse Bro messages
* org.apache.metron.parsers.sourcefire.BasicSourcefireParser : Parse Sourcefire messages
* org.apache.metron.parsers.lancope.BasicLancopeParser : Parse Lancope messages
+* org.apache.metron.parsers.syslog.Syslog5424Parser : Parse Syslog RFC 5424 messages
### Grok Parser Adapters
Grok parser adapters are designed primarly for someone who is not a Java coder for quickly standing up a parser adapter for lower velocity topologies. Grok relies on Regex for message parsing, which is much slower than purpose-built Java parsers, but is more extensible. Grok parsers are defined via a config file and the topplogy does not need to be recombiled in order to make changes to them. An example of a Grok perser is:
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index e586492..5077791 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -261,6 +261,11 @@
<artifactId>json-path</artifactId>
<version>2.3.0</version>
</dependency>
+ <dependency>
+ <groupId>com.github.palindromicity</groupId>
+ <artifactId>simple-syslog-5424</artifactId>
+ <version>${global_simple_syslog_version}</version>
+ </dependency>
</dependencies>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
new file mode 100644
index 0000000..5f62692
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/syslog5424.json
@@ -0,0 +1,7 @@
+{
+ "parserClassName":"org.apache.metron.parsers.syslog.Syslog5424Parser",
+ "sensorTopic":"syslog5424",
+ "parserConfig": {
+ "nilPolicy": "DASH"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
new file mode 100644
index 0000000..e3ad941
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/syslog/Syslog5424Parser.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.syslog;
+
+import com.github.palindromicity.syslog.AllowableDeviations;
+import com.github.palindromicity.syslog.NilPolicy;
+import com.github.palindromicity.syslog.SyslogParser;
+import com.github.palindromicity.syslog.SyslogParserBuilder;
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.parsers.BasicParser;
+import org.json.simple.JSONObject;
+
+import java.io.BufferedReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * Parser for well structured RFC 5424 messages.
+ */
+public class Syslog5424Parser extends BasicParser {
+ public static final String NIL_POLICY_CONFIG = "nilPolicy";
+ private transient SyslogParser syslogParser;
+
+ @Override
+ public void configure(Map<String, Object> config) {
+ // Default to OMIT policy for nil fields
+ // this means they will not be in the returned field set
+ String nilPolicyStr = (String) config.getOrDefault(NIL_POLICY_CONFIG, NilPolicy.OMIT.name());
+ NilPolicy nilPolicy = NilPolicy.valueOf(nilPolicyStr);
+ syslogParser = new SyslogParserBuilder()
+ .withNilPolicy(nilPolicy)
+ .withDeviations(EnumSet.of(AllowableDeviations.PRIORITY,AllowableDeviations.VERSION))
+ .build();
+ }
+
+ @Override
+ public void init() {
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public List<JSONObject> parse(byte[] rawMessage) {
+ try {
+ if (rawMessage == null || rawMessage.length == 0) {
+ return null;
+ }
+
+ String originalString = new String(rawMessage);
+ List<JSONObject> returnList = new ArrayList<>();
+ try (Reader reader = new BufferedReader(new StringReader(originalString))) {
+ syslogParser.parseLines(reader, (m) -> {
+ JSONObject jsonObject = new JSONObject(m);
+ // be sure to put in the original string, and the timestamp.
+ // we wil just copy over the timestamp from the syslog
+ jsonObject.put("original_string", originalString);
+ setTimestamp(jsonObject);
+ returnList.add(jsonObject);
+ });
+
+ return returnList;
+ }
+ } catch (Exception e) {
+ String message = "Unable to parse " + new String(rawMessage) + ": " + e.getMessage();
+ LOG.error(message, e);
+ throw new IllegalStateException(message, e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void setTimestamp(JSONObject message) {
+ String timeStampString = (String) message.get(SyslogFieldKeys.HEADER_TIMESTAMP.getField());
+ if (!StringUtils.isBlank(timeStampString) && !timeStampString.equals("-")) {
+ message.put("timestamp", timeStampString);
+ } else {
+ message.put("timestamp", LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
new file mode 100644
index 0000000..c9c47ce
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/Syslog5424ParserIntegrationTest.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.integration;
+
+import org.apache.metron.parsers.integration.validation.SampleDataValidation;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class Syslog5424ParserIntegrationTest extends ParserIntegrationTest {
+ @Override
+ String getSensorType() {
+ return "syslog5424";
+ }
+
+ @Override
+ List<ParserValidation> getValidations() {
+ return new ArrayList<ParserValidation>() {{
+ add(new SampleDataValidation());
+ }};
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
new file mode 100644
index 0000000..0ef26ff
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/syslog/Syslog5424ParserTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.syslog;
+
+import com.github.palindromicity.syslog.NilPolicy;
+import com.github.palindromicity.syslog.dsl.SyslogFieldKeys;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+public class Syslog5424ParserTest {
+ private static final String SYSLOG_LINE_ALL = "<14>1 2014-06-20T09:14:07+00:00 loggregator"
+ + " d0602076-b14a-4c55-852a-981e7afeed38 DEA MSG-01"
+ + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+ + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance";
+
+ private static final String SYSLOG_LINE_MISSING = "<14>1 2014-06-20T09:14:07+00:00 loggregator"
+ + " d0602076-b14a-4c55-852a-981e7afeed38 DEA -"
+ + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+ + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance";
+
+ private static final String SYSLOG_LINE_MISSING_DATE = "<14>1 - loggregator"
+ + " d0602076-b14a-4c55-852a-981e7afeed38 DEA -"
+ + " [exampleSDID@32473 iut=\"3\" eventSource=\"Application\" eventID=\"1011\"]"
+ + " [exampleSDID@32480 iut=\"4\" eventSource=\"Other Application\" eventID=\"2022\"] Removing instance";
+
+ private static final String expectedVersion = "1";
+ private static final String expectedMessage = "Removing instance";
+ private static final String expectedAppName = "d0602076-b14a-4c55-852a-981e7afeed38";
+ private static final String expectedHostName = "loggregator";
+ private static final String expectedPri = "14";
+ private static final String expectedFacility = "1";
+ private static final String expectedSeverity = "6";
+ private static final String expectedProcId = "DEA";
+ private static final String expectedTimestamp = "2014-06-20T09:14:07+00:00";
+ private static final String expectedMessageId = "MSG-01";
+
+ private static final String expectedIUT1 = "3";
+ private static final String expectedIUT2 = "4";
+ private static final String expectedEventSource1 = "Application";
+ private static final String expectedEventSource2 = "Other Application";
+ private static final String expectedEventID1 = "1011";
+ private static final String expectedEventID2 = "2022";
+
+ @Test
+ public void testHappyPath() {
+ test(null, SYSLOG_LINE_ALL, (message) -> Assert.assertEquals(expectedMessageId, message.get(SyslogFieldKeys.HEADER_MSGID.getField())));
+ }
+
+ @Test
+ public void testOmit() {
+ test(NilPolicy.OMIT, SYSLOG_LINE_MISSING, (message) -> Assert.assertFalse(message.containsKey(SyslogFieldKeys.HEADER_MSGID)));
+ }
+
+ @Test
+ public void testDash() {
+ test(NilPolicy.DASH, SYSLOG_LINE_MISSING, (message) -> Assert.assertEquals("-", message.get(SyslogFieldKeys.HEADER_MSGID.getField())));
+ }
+
+ @Test()
+ public void testNull() {
+ test(NilPolicy.NULL, SYSLOG_LINE_MISSING, (message) -> {
+ Assert.assertTrue(message.containsKey(SyslogFieldKeys.HEADER_MSGID.getField()));
+ Assert.assertNull(message.get(SyslogFieldKeys.HEADER_MSGID.getField()));
+ });
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testNotValid() {
+ test(null, "not valid", (message) -> Assert.assertTrue(false));
+ }
+
+ public void test(NilPolicy nilPolicy, String line, Consumer<JSONObject> msgIdChecker) {
+ Syslog5424Parser parser = new Syslog5424Parser();
+ Map<String, Object> config = new HashMap<>();
+ if (nilPolicy != null) {
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, nilPolicy.name());
+ }
+ parser.configure(config);
+
+ List<JSONObject> output = parser.parse(line.getBytes());
+ }
+
+ @Test
+ public void testReadMultiLine() throws Exception {
+ Syslog5424Parser parser = new Syslog5424Parser();
+ Map<String, Object> config = new HashMap<>();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
+ parser.configure(config);
+ StringBuilder builder = new StringBuilder();
+ builder
+ .append(SYSLOG_LINE_ALL)
+ .append("\n")
+ .append(SYSLOG_LINE_MISSING)
+ .append("\n")
+ .append(SYSLOG_LINE_ALL);
+ List<JSONObject> output = parser.parse(builder.toString().getBytes());
+ Assert.assertEquals(3,output.size());
+ }
+
+ @Test
+ public void testMissingTimestamp() {
+ Syslog5424Parser parser = new Syslog5424Parser();
+ Map<String, Object> config = new HashMap<>();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.DASH.name());
+ parser.configure(config);
+ List<JSONObject> output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+ String timeStampString = output.get(0).get("timestamp").toString();
+ DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+ config.clear();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.NULL.name());
+ parser.configure(config);
+ output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+ timeStampString = output.get(0).get("timestamp").toString();
+ DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+
+ config.clear();
+ config.put(Syslog5424Parser.NIL_POLICY_CONFIG, NilPolicy.OMIT.name());
+ parser.configure(config);
+ output = parser.parse(SYSLOG_LINE_MISSING_DATE.getBytes());
+ timeStampString = output.get(0).get("timestamp").toString();
+ DateTimeFormatter.ISO_DATE_TIME.parse(timeStampString);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/ff1f9cf5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1c1fa3d..9bde04e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,6 +137,7 @@
<global_reflections_version>0.9.10</global_reflections_version>
<global_checkstyle_version>8.0</global_checkstyle_version>
<global_log4j_core_version>2.1</global_log4j_core_version>
+ <global_simple_syslog_version>0.0.8</global_simple_syslog_version>
<global_spark_version>2.3.1</global_spark_version>
</properties>