You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2019/05/20 20:27:56 UTC
[metron] branch master updated: METRON-2118 Added a LEEF parser
(simonellistonball via mmiklavc) closes apache/metron#1408
This is an automated email from the ASF dual-hosted git repository.
mmiklavcic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 22cc622 METRON-2118 Added a LEEF parser (simonellistonball via mmiklavc) closes apache/metron#1408
22cc622 is described below
commit 22cc622d6f08cfa77c06dea2df7f4c56edc1862e
Author: simonellistonball <si...@simonellistonball.com>
AuthorDate: Mon May 20 14:25:41 2019 -0600
METRON-2118 Added a LEEF parser (simonellistonball via mmiklavc) closes apache/metron#1408
---
.../src/main/sample/data/leef/parsed/leefParsed | 4 +
.../src/main/sample/data/leef/raw/leefOutput | 4 +
metron-platform/metron-parsing/README.md | 2 +
.../src/main/config/zookeeper/parsers/leef.json | 4 +
.../org/apache/metron/parsers/utils/DateUtils.java | 9 +
.../parsers/integration/ParserIntegrationTest.java | 3 +-
.../org/apache/metron/parsers/cef/CEFParser.java | 97 ++++---
.../org/apache/metron/parsers/leef/LEEFParser.java | 287 +++++++++++++++++++++
.../apache/metron/parsers/cef/CEFParserTest.java | 57 ++--
.../apache/metron/parsers/leef/LEEFParserTest.java | 243 +++++++++++++++++
.../org/apache/metron/parsers/leef/sample.leef | 4 +
.../org/apache/metron/parsers/leef/sample.schema | 27 ++
12 files changed, 659 insertions(+), 82 deletions(-)
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/leef/parsed/leefParsed b/metron-platform/metron-integration-test/src/main/sample/data/leef/parsed/leefParsed
new file mode 100644
index 0000000..0eb5949
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/leef/parsed/leefParsed
@@ -0,0 +1,4 @@
+{"msg":"Alert: CPUWarning Threshold Exceeded\\nSubject: 10.201.114.164\\nSeverity:Warning","DeviceEvent":"192","sev":"3","TrendMicroDsTenant":"Primary","DeviceVersion":"<DSA version>","original_string":"LEEF:2.0|Trend Micro|Deep Security Manager|<DSA version>|192|cat=System\tname=Alert Ended\tdesc=Alert: CPU Warning Threshold Exceeded\\nSubject: 10.201.114.164\\nSeverity: Warning\tsev=3\tsrc=10.201.114.164\tusrName=System\tmsg=Alert: CPUWarning Threshold Exceeded\\nSubject: 10.201.114.16 [...]
+{"DeviceEvent":"2002779","TrendMicroDsTenantId":"0","sev":"8 cn1","TrendMicroDsTenant":"Primary","DeviceVersion":"<DSA version>","act":"updated","original_string":"LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|2002779|cat=Integrity Monitor\tname=Microsoft Windows - System file modified\tdesc=Microsoft Windows - System file modified\tsev=8 cn1=37 cn1Label=Host ID\tdvchost=www.example.com\tTrendMicroDsTenant=Primary\tTrendMicroDsTenantId=0\tact=updated\tsuser=admin","suser":"admin [...]
+{"msg":"Realtime","DeviceEvent":"4000030","TrendMicroDsTenantId":"0","sev":"6 cn1","TrendMicroDsTenant":"Primary","filePath":"C:\\\\Windows\\\\System32\\\\virus.exe","DeviceVersion":"<DSA version>","TrendMicroDsTags":"FS","TrendMicroDsMalwareTarget":"Multiple","dvc":"10.0.0.1","TrendMicroDsRelevantDetectionNames":"Ransom_CERBER.BZC;Ransom_CERBER.C;Ransom_CRYPNISCA.SM","act":"Terminate","original_string":"LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|4000030|cat=Anti-Malware\tnam [...]
+{"msg":"Suspicious","DeviceEvent":"5000000","request":"http:\/\/yw.olx5x9ny.org.it\/HvuauRH\/eighgSS.htm","TrendMicroDsTenantId":"0","sev":"6 cn1","TrendMicroDsTenant":"Primary","DeviceVersion":"<DSA version>","original_string":"LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|5000000|cat=Web Reputation\tname=WebReputation\tdesc=WebReputation\tsev=6 cn1=3 cn1Label=Host ID\tdvchost=exch01.example.com\tTrendMicroDsTenant=Primary\tTrendMicroDsTenantId=0\trequest=http:\/\/yw.olx5x9ny.o [...]
\ No newline at end of file
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/leef/raw/leefOutput b/metron-platform/metron-integration-test/src/main/sample/data/leef/raw/leefOutput
new file mode 100644
index 0000000..41af31c
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/leef/raw/leefOutput
@@ -0,0 +1,4 @@
+LEEF:2.0|Trend Micro|Deep Security Manager|<DSA version>|192|cat=System name=Alert Ended desc=Alert: CPU Warning Threshold Exceeded\nSubject: 10.201.114.164\nSeverity: Warning sev=3 src=10.201.114.164 usrName=System msg=Alert: CPUWarning Threshold Exceeded\nSubject: 10.201.114.164\nSeverity:Warning TrendMicroDsTenant=Primary
+LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|2002779|cat=Integrity Monitor name=Microsoft Windows - System file modified desc=Microsoft Windows - System file modified sev=8 cn1=37 cn1Label=Host ID dvchost=www.example.com TrendMicroDsTenant=Primary TrendMicroDsTenantId=0 act=updated suser=admin
+LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|4000030|cat=Anti-Malware name=HEU_AEGIS_CRYPT desc=HEU_AEGIS_CRYPT sev=6 cn1=241 cn1Label=Host ID dvc=10.0.0.1 TrendMicroDsTags=FS TrendMicroDsTenant=Primary TrendMicroDsTenantId=0 filePath=C:\\Windows\\System32\\virus.exe act=Terminate msg=Realtime TrendMicroDsMalwareTarget=Multiple TrendMicroDsMalwareTargetType=File System TrendMicroDsFileMD5=1947A1BC0982C5871FA3768CD025453E#011 TrendMicroDsFileSHA1=5AD084DDCD8F80FBF2EE3F0E4F812E81 [...]
+LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|5000000|cat=Web Reputation name=WebReputation desc=WebReputation sev=6 cn1=3 cn1Label=Host ID dvchost=exch01.example.com TrendMicroDsTenant=Primary TrendMicroDsTenantId=0 request=http://yw.olx5x9ny.org.it/HvuauRH/eighgSS.htm msg=Suspicious
\ No newline at end of file
diff --git a/metron-platform/metron-parsing/README.md b/metron-platform/metron-parsing/README.md
index 085fb33..3c6d26a 100644
--- a/metron-platform/metron-parsing/README.md
+++ b/metron-platform/metron-parsing/README.md
@@ -655,6 +655,8 @@ Java parser adapters are intended for higher-velocity topologies and are not eas
* org.apache.metron.parsers.lancope.BasicLancopeParser : Parse Lancope messages
* org.apache.metron.parsers.syslog.Syslog5424Parser : Parse Syslog RFC 5424 messages
* org.apache.metron.parsers.syslog.Syslog3164Parser : Parse Syslog RFC 3164 messages
+* org.apache.metron.parsers.cef.CEFParser: Parse CEF format messages
+* org.apache.metron.parsers.leef.LEEFParser: Parse LEEF format messages
### Grok Parser Adapters
Grok parser adapters are designed primarily for someone who is not a Java coder for quickly standing up a parser adapter for lower velocity topologies. Grok relies on Regex for message parsing, which is much slower than purpose-built Java parsers, but is more extensible. Grok parsers are defined via a config file and the topplogy does not need to be recompiled in order to make changes to them. Example of a Grok parsers are:
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/leef.json b/metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/leef.json
new file mode 100644
index 0000000..4f3998b
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/config/zookeeper/parsers/leef.json
@@ -0,0 +1,4 @@
+{
+ "parserClassName":"org.apache.metron.parsers.leef.LEEFParser",
+ "sensorTopic":"leef"
+}
\ No newline at end of file
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/DateUtils.java b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
index 888649a..f2a2fb7 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
@@ -36,6 +36,15 @@ import org.apache.commons.lang.StringUtils;
*/
public class DateUtils {
+ // Per IBM LEEF guide at https://www.ibm.com/support/knowledgecenter/SS42VS_DSM/c_LEEF_Format_Guide_intro.html
+ public static List<SimpleDateFormat> DATE_FORMATS_LEEF = new ArrayList<SimpleDateFormat>() {
+ {
+ add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss.SSS zzz"));
+ add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss.SSS"));
+ add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss"));
+ }
+ };
+
public static List<SimpleDateFormat> DATE_FORMATS_CEF = new ArrayList<SimpleDateFormat>() {
{
// as per CEF Spec
diff --git a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
index f14e260..29ec1fe 100644
--- a/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
+++ b/metron-platform/metron-parsing/metron-parsers-common/src/test/java/org/apache/metron/parsers/integration/ParserIntegrationTest.java
@@ -45,7 +45,8 @@ public abstract class ParserIntegrationTest extends BaseIntegrationTest {
"websphere",
"yaf",
"syslog3164",
- "syslog5424"
+ "syslog5424",
+ "leef"
);
protected List<byte[]> inputMessages;
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
index 03de37c..a807f13 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
@@ -45,7 +45,7 @@ public class CEFParser extends BasicParser {
private static final Charset UTF_8 = Charset.forName("UTF-8");
private Pattern p;
- private Pattern pext;
+ private static final Pattern patternExtensions = Pattern.compile(EXTENSION_CAPTURE_PATTERN);
public void init() {
@@ -94,8 +94,50 @@ public class CEFParser extends BasicParser {
p = Pattern.compile(pattern);
- // key finder for extensions
- pext = Pattern.compile(EXTENSION_CAPTURE_PATTERN);
+ }
+
+ public static void parseExtensions(String ext, JSONObject obj) {
+ Matcher m = patternExtensions.matcher(ext);
+
+ int index = 0;
+ String key = null;
+ String value = null;
+ Map<String, String> labelMap = new HashMap<String, String>();
+
+ while (m.find()) {
+ if (key == null) {
+ key = ext.substring(index, m.start());
+ index = m.end();
+ if (!m.find()) {
+ break;
+ }
+ }
+ value = ext.substring(index, m.start());
+ index = m.end();
+ int v = value.lastIndexOf(" ");
+ if (v > 0) {
+ String temp = value.substring(0, v).trim();
+ if (key.endsWith("Label")) {
+ labelMap.put(key.substring(0, key.length() - 5), temp);
+ } else {
+ obj.put(key, temp);
+ }
+ key = value.substring(v).trim();
+ }
+ }
+ value = ext.substring(index);
+
+ // Build a map of Label extensions to apply later
+ if (key.endsWith("Label")) {
+ labelMap.put(key.substring(0, key.length() - 5), value);
+ } else {
+ obj.put(key, value);
+ }
+
+ // Apply the labels to custom fields
+ for (Entry<String, String> label : labelMap.entrySet()) {
+ mutate(obj, label.getKey(), label.getValue());
+ }
}
@SuppressWarnings("unchecked")
@@ -109,7 +151,7 @@ public class CEFParser extends BasicParser {
while (matcher.find()) {
JSONObject obj = new JSONObject();
if (matcher.matches()) {
- LOG.info("Found %d groups", matcher.groupCount());
+ LOG.debug("Found %d groups", matcher.groupCount());
obj.put("DeviceVendor", matcher.group("DeviceVendor"));
obj.put("DeviceProduct", matcher.group("DeviceProduct"));
obj.put("DeviceVersion", matcher.group("DeviceVersion"));
@@ -118,48 +160,7 @@ public class CEFParser extends BasicParser {
obj.put("Severity", standardizeSeverity(matcher.group("Severity")));
}
- String ext = matcher.group("extensions");
- Matcher m = pext.matcher(ext);
-
- int index = 0;
- String key = null;
- String value = null;
- Map<String, String> labelMap = new HashMap<String, String>();
-
- while (m.find()) {
- if (key == null) {
- key = ext.substring(index, m.start());
- index = m.end();
- if (!m.find()) {
- break;
- }
- }
- value = ext.substring(index, m.start());
- index = m.end();
- int v = value.lastIndexOf(" ");
- if (v > 0) {
- String temp = value.substring(0, v).trim();
- if (key.endsWith("Label")) {
- labelMap.put(key.substring(0, key.length() - 5), temp);
- } else {
- obj.put(key, temp);
- }
- key = value.substring(v).trim();
- }
- }
- value = ext.substring(index);
-
- // Build a map of Label extensions to apply later
- if (key.endsWith("Label")) {
- labelMap.put(key.substring(0, key.length() - 5), value);
- } else {
- obj.put(key, value);
- }
-
- // Apply the labels to custom fields
- for (Entry<String, String> label : labelMap.entrySet()) {
- mutate(obj, label.getKey(), label.getValue());
- }
+ parseExtensions(matcher.group("extensions"), obj);
// Rename standard CEF fields to comply with Metron standards
obj = mutate(obj, "dst", "ip_dst_addr");
@@ -259,12 +260,10 @@ public class CEFParser extends BasicParser {
@Override
public void configure(Map<String, Object> config) {
- // TODO Auto-generated method stub
-
}
@SuppressWarnings("unchecked")
- private JSONObject mutate(JSONObject json, String oldKey, String newKey) {
+ private static JSONObject mutate(JSONObject json, String oldKey, String newKey) {
if (json.containsKey(oldKey)) {
json.put(newKey, json.remove(oldKey));
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
new file mode 100644
index 0000000..db19ebd
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsers/src/main/java/org/apache/metron/parsers/leef/LEEFParser.java
@@ -0,0 +1,287 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.leef;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.text.SimpleDateFormat;
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.metron.common.Constants.Fields;
+import org.apache.metron.parsers.BasicParser;
+import org.apache.metron.parsers.DefaultMessageParserResult;
+import org.apache.metron.parsers.ParseException;
+import org.apache.metron.parsers.cef.CEFParser;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.apache.metron.parsers.utils.DateUtils;
+import org.apache.metron.parsers.utils.SyslogUtils;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * LEEF Parser
+ *
+ * Based primarily on the IBM LEEF guide at https://www.ibm.com/support/knowledgecenter/SS42VS_DSM/c_LEEF_Format_Guide_intro.html
+ * with support for other found in the wild examples shown in tests.
+ *
+ */
+public class LEEFParser extends BasicParser {
+ private static final long serialVersionUID = 1L;
+
+ public enum HeaderFields {
+ DEVICE_VENDOR("DeviceVendor"),
+ DEVICE_PRODUCT("DeviceProduct"),
+ DEVICE_VERSION("DeviceVersion"),
+ DEVICE_EVENT("DeviceEvent"),
+ DELIMITER("Delimiter"),
+ VERSION("Version")
+ ;
+
+ private String name;
+
+ HeaderFields(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+
+ // Field name for custom device time in LEEF
+ private static final String DEV_TIME = "devTime";
+ private static final String DEV_TIME_FORMAT = "devTimeFormat";
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*";
+ private static final Charset UTF_8 = StandardCharsets.UTF_8;
+
+ private Pattern pattern;
+
+ public void init() {
+
+ // LEEF Headers: Version|Device Vendor|Device Product|Device Version|Device Event|Delimiter
+ String syslogTime = "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b +(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?!<[0-9])(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])(?::(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?))(?![0-9])?";
+ String syslogTime5424 = "(?:\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2}))";
+ String syslogPriority = "<(?:[0-9]+)>";
+ String syslogHost = "[a-z0-9\\.\\\\-_]+";
+
+ StringBuilder sb = new StringBuilder("");
+ sb.append("(?<syslogPriority>");
+ sb.append(syslogPriority);
+ sb.append(")?");
+ sb.append("(?<syslogTime>");
+ sb.append(syslogTime);
+ sb.append("|");
+ sb.append(syslogTime5424);
+ sb.append(")?");
+
+ sb.append("(?<syslogHost>");
+ sb.append(syslogHost);
+ sb.append(")?");
+
+ sb.append(".*");
+
+ sb.append("LEEF:(?<");
+ sb.append(HeaderFields.VERSION.getName());
+ sb.append(">1.0|2.0|0)?\\|");
+
+ headerBlock(HeaderFields.DEVICE_VENDOR.getName(), sb);
+ sb.append("\\|");
+ headerBlock(HeaderFields.DEVICE_PRODUCT.getName(), sb);
+ sb.append("\\|");
+ headerBlock(HeaderFields.DEVICE_VERSION.getName(), sb);
+ sb.append("\\|");
+ headerBlock(HeaderFields.DEVICE_EVENT.getName(), sb);
+ sb.append("\\|");
+
+ // add optional delimiter header (only applicable for LEEF 2.0)
+ sb.append("(");
+ headerBlock(HeaderFields.DELIMITER.getName(), sb);
+ sb.append("\\|");
+ sb.append(")?");
+
+ // extension capture:
+ sb.append(" ?(?<extensions>.*)");
+ pattern = Pattern.compile(sb.toString());
+ }
+
+ public Optional<MessageParserResult<JSONObject>> parseOptionalResult(byte[] rawMessage) {
+ List<JSONObject> messages = new ArrayList<>();
+ Map<Object,Throwable> errors = new HashMap<>();
+ String originalMessage = null;
+
+ try (BufferedReader reader = new BufferedReader(new StringReader(new String(rawMessage, StandardCharsets.UTF_8)))) {
+ while ((originalMessage = reader.readLine()) != null) {
+ Matcher matcher = pattern.matcher(originalMessage);
+ while (matcher.find()) {
+ JSONObject obj = new JSONObject();
+ if (!matcher.matches()) {
+ break;
+ }
+ LOG.debug("Found %d groups", matcher.groupCount());
+ obj.put(HeaderFields.DEVICE_VENDOR.getName(),
+ matcher.group(HeaderFields.DEVICE_VENDOR.getName()));
+ obj.put(HeaderFields.DEVICE_PRODUCT.getName(),
+ matcher.group(HeaderFields.DEVICE_PRODUCT.getName()));
+ obj.put(HeaderFields.DEVICE_VERSION.getName(),
+ matcher.group(HeaderFields.DEVICE_VERSION.getName()));
+ obj.put(HeaderFields.DEVICE_EVENT.getName(),
+ matcher.group(HeaderFields.DEVICE_EVENT.getName()));
+
+ String ext = matcher.group("extensions");
+
+ // In LEEF 2.0 the delimiter can be specified
+ String version = matcher.group(HeaderFields.VERSION.getName());
+ if (version.equals("2.0")) {
+ String delimiter = matcher.group(HeaderFields.DELIMITER.getName());
+ if (delimiter == null || delimiter.length() == 0) {
+ delimiter = "\\t";
+ }
+ delimiter = "(?<!\\\\)[" + delimiter.replace("^", "\\^").replace("\t", "\\t") + "]";
+
+ String[] kvs = ext.split(delimiter);
+ for (String kv : kvs) {
+ String[] a = kv.split("=");
+ obj.put(a[0], a[1]);
+ }
+ } else if (version.equals("1.0") || version.isEmpty()) {
+ String delimiter = "\t";
+ String[] kvs = ext.split(delimiter);
+ for (String kv : kvs) {
+ String[] a = kv.split("=");
+ obj.put(a[0], a[1]);
+ }
+ } else {
+ // Found in the wild examples using CEF rules, which need to handle the processing per the CEFParser
+ // Note that technically LEEF does not support the CEF approach to numbered custom variables.
+ // We however do here, due to some found in the wild exceptions to the standard.
+ CEFParser.parseExtensions(ext, obj);
+ }
+
+ // Rename standard CEF fields to comply with Metron standards
+ obj = mutate(obj, "dst", Fields.DST_ADDR.getName());
+ obj = mutate(obj, "dstPort", Fields.DST_PORT.getName());
+ obj = convertToInt(obj, Fields.DST_PORT.getName());
+
+ obj = mutate(obj, "src", Fields.SRC_ADDR.getName());
+ obj = mutate(obj, "srcPort", Fields.SRC_PORT.getName());
+ obj = convertToInt(obj, Fields.SRC_PORT.getName());
+
+ obj.put(Fields.ORIGINAL.getName(), originalMessage);
+
+ // add the host
+ String host = matcher.group("syslogHost");
+ if (!(host == null || host.isEmpty())) {
+ obj.put("host", host);
+ }
+
+ // apply timestamp from message if present, using devTime, syslog
+ // timestamp,
+ // default to current system time
+ //devTime, devTimeFormat, calLanguage, calCountryOrRegion
+ if (obj.containsKey(DEV_TIME)) {
+ String devTime = (String) obj.get(DEV_TIME);
+ try {
+ // DateFormats allowed in LEEF
+ // epoch
+ // MMM dd yyyy HH:mm:ss
+ // MMM dd yyyy HH:mm:ss.SSS
+ // MMM dd yyyy HH:mm:ss.SSS zzz
+ // custom in devTimeFormat field
+ final String devTimeFormat = (String) obj.get(DEV_TIME_FORMAT);
+
+ List<SimpleDateFormat> formats = (obj.containsKey(DEV_TIME_FORMAT)) ?
+ new ArrayList<SimpleDateFormat>() {{
+ add(new SimpleDateFormat(devTimeFormat));
+ }} :
+ DateUtils.DATE_FORMATS_LEEF;
+ obj.put(Fields.TIMESTAMP.getName(), DateUtils.parseMultiformat(devTime, formats));
+ } catch (java.text.ParseException e) {
+ errors.put(originalMessage,
+ new IllegalStateException("devTime field present in LEEF but cannot be parsed",
+ e));
+ continue;
+ }
+ } else {
+ String logTimestamp = matcher.group("syslogTime");
+ if (!(logTimestamp == null || logTimestamp.isEmpty())) {
+ try {
+ obj.put(Fields.TIMESTAMP.getName(),
+ SyslogUtils.parseTimestampToEpochMillis(logTimestamp, Clock.systemUTC()));
+ } catch (ParseException e) {
+ errors.put(originalMessage,
+ new IllegalStateException("Cannot parse syslog timestamp", e));
+ continue;
+ }
+ } else {
+ obj.put(Fields.TIMESTAMP.getName(), System.currentTimeMillis());
+ }
+ }
+ messages.add(obj);
+ }
+ }
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ Exception innerException = new IllegalStateException("LEEF parser Error: "
+ + e.getMessage()
+ + " on "
+ + originalMessage, e);
+ return Optional.of(new DefaultMessageParserResult<>(innerException));
+ }
+ return Optional.of(new DefaultMessageParserResult<>(messages, errors));
+ }
+
+ @SuppressWarnings("unchecked")
+ private JSONObject convertToInt(JSONObject obj, String key) {
+ if (obj.containsKey(key)) {
+ obj.put(key, Integer.valueOf((String) obj.get(key)));
+ }
+ return obj;
+ }
+
+ private void headerBlock(String name, StringBuilder sb) {
+ sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")");
+ }
+
+ @Override
+ public void configure(Map<String, Object> config) {
+ }
+
+ @SuppressWarnings("unchecked")
+ private JSONObject mutate(JSONObject json, String oldKey, String newKey) {
+ if (json.containsKey(oldKey)) {
+ json.put(newKey, json.remove(oldKey));
+ }
+ return json;
+ }
+
+}
+
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
index f679c91..e6ca142 100644
--- a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
@@ -21,12 +21,14 @@ package org.apache.metron.parsers.cef;
import java.io.IOException;
import java.net.URL;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
+import org.apache.metron.common.Constants.Fields;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
@@ -45,7 +47,7 @@ import com.google.common.io.Resources;
public class CEFParserTest {
- private static final Charset UTF_8 = Charset.forName("utf-8");
+ private static final Charset UTF_8 = StandardCharsets.UTF_8;
private CEFParser parser;
@Before
@@ -64,9 +66,9 @@ public class CEFParserTest {
public void testEscaping() {
for (JSONObject obj : parse(
"Sep 19 08:26:10 host CEF:0|security|threatmanager|1.0|100|detected a \\ in packet|10|src=10.0.0.1 act=blocked a \\ dst=1.1.1.1")) {
- Assert.assertEquals("10.0.0.1", obj.get("ip_src_addr"));
+ Assert.assertEquals("10.0.0.1", obj.get(Fields.SRC_ADDR.getName()));
Assert.assertEquals("blocked a \\", obj.get("deviceAction"));
- Assert.assertEquals("1.1.1.1", obj.get("ip_dst_addr"));
+ Assert.assertEquals("1.1.1.1", obj.get(Fields.DST_ADDR.getName()));
}
}
@@ -87,9 +89,9 @@ public class CEFParserTest {
public void testBasicExtensions() {
for (JSONObject obj : parse(
"CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
- Assert.assertEquals("10.0.0.1", obj.get("ip_src_addr"));
- Assert.assertEquals("2.1.2.2", obj.get("ip_dst_addr"));
- Assert.assertEquals(1232, obj.get("ip_src_port"));
+ Assert.assertEquals("10.0.0.1", obj.get(Fields.SRC_ADDR.getName()));
+ Assert.assertEquals("2.1.2.2", obj.get(Fields.DST_ADDR.getName()));
+ Assert.assertEquals(1232, obj.get(Fields.SRC_PORT.getName()));
}
}
@@ -111,22 +113,22 @@ public class CEFParserTest {
for (JSONObject obj : parse(
"CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt=May 1 2016 09:29:11.356 -0400 dst=2.1.2.2 spt=1232")) {
- Assert.assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp")));
- Assert.assertEquals(correctTime, obj.get("timestamp"));
+ Assert.assertEquals(new Date(correctTime), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
}
for (JSONObject obj : parse(
"2016-06-01T09:29:11.356-04:00 host CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt=May 1 2016 09:29:11.356 -0400 dst=2.1.2.2 spt=1232")) {
- Assert.assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp")));
- Assert.assertEquals(correctTime, obj.get("timestamp"));
+ Assert.assertEquals(new Date(correctTime), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
}
for (JSONObject obj : parse(
"2016-05-01T09:29:11.356-04:00 host CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
- Assert.assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp")));
- Assert.assertEquals(correctTime, obj.get("timestamp"));
+ Assert.assertEquals(new Date(correctTime), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
}
for (JSONObject obj : parse(
"CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
- Assert.assertNotNull(obj.get("timestamp"));
+ Assert.assertNotNull(obj.get(Fields.TIMESTAMP.getName()));
}
}
@@ -137,8 +139,8 @@ public class CEFParserTest {
.getTime();
for (JSONObject obj : parse("CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt="
+ String.valueOf(correctTime) + " dst=2.1.2.2 spt=1232")) {
- Assert.assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp")));
- Assert.assertEquals(correctTime, obj.get("timestamp"));
+ Assert.assertEquals(new Date(correctTime), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
}
}
@@ -146,8 +148,8 @@ public class CEFParserTest {
SimpleDateFormat sdf = new SimpleDateFormat("MMM dd HH:mm:ss.SSS");
for (JSONObject obj : parse("CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt="
+ sdf.format(input.getTime()) + " dst=2.1.2.2 spt=1232")) {
- Assert.assertEquals(expected.getTimeInMillis(), obj.get("timestamp"));
- Assert.assertEquals(expected.getTime(), new Date((long) obj.get("timestamp")));
+ Assert.assertEquals(expected.getTimeInMillis(), obj.get(Fields.TIMESTAMP.getName()));
+ Assert.assertEquals(expected.getTime(), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
}
}
@@ -209,23 +211,14 @@ public class CEFParserTest {
for (String inputString : lines) {
JSONObject parsed = parse(inputString).get(0);
Assert.assertNotNull(parsed);
- Assert.assertNotNull(parsed.get("timestamp"));
- Assert.assertTrue((long) parsed.get("timestamp") > 0);
+ Assert.assertNotNull(parsed.get(Fields.TIMESTAMP.getName()));
+ Assert.assertTrue((long) parsed.get(Fields.TIMESTAMP.getName()) > 0);
JSONParser parser = new JSONParser();
Map<?, ?> json = null;
- try {
- json = (Map<?, ?>) parser.parse(parsed.toJSONString());
- Assert.assertEquals(true, validateJsonData(schema, json.toString()));
- } catch (ParseException e) {
- e.printStackTrace();
- }
-
- // test against an explicit json example
- if (!targetJson.isEmpty()) {
-
- }
+ json = (Map<?, ?>) parser.parse(parsed.toJSONString());
+ Assert.assertEquals(true, validateJsonData(schema, json.toString()));
}
}
@@ -258,7 +251,7 @@ public class CEFParserTest {
JSONObject obj = parse.get(0);
Assert.assertEquals("TestVendor", obj.get("DeviceVendor"));
- Assert.assertEquals(1423441663000L, obj.get("timestamp"));
+ Assert.assertEquals(1423441663000L, obj.get(Fields.TIMESTAMP.getName()));
Assert.assertEquals("9223372036854775807", obj.get("Test Long"));
Assert.assertEquals(obj.get("Test FP Number"), String.valueOf(1.234F));
Assert.assertEquals("00:00:0c:07:ac:00", obj.get("smac"));
@@ -283,7 +276,7 @@ public class CEFParserTest {
}
private List<JSONObject> parse(String string) {
- List<JSONObject> parse = parser.parse(string.getBytes(Charset.forName("utf-8")));
+ List<JSONObject> parse = parser.parse(string.getBytes(UTF_8));
Assert.assertNotNull(parse);
return parse;
}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
new file mode 100644
index 0000000..704d748
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/java/org/apache/metron/parsers/leef/LEEFParserTest.java
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.parsers.leef;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import org.apache.metron.common.Constants.Fields;
+import org.apache.metron.parsers.interfaces.MessageParserResult;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+import com.google.common.io.Resources;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+public class LEEFParserTest {
+ private static final Charset UTF_8 = StandardCharsets.UTF_8;
+ private LEEFParser parser;
+
+ @Before
+ public void setUp() {
+ parser = new LEEFParser();
+ parser.init();
+ }
+
+ @Test
+ public void testInvalid() {
+ List<JSONObject> obj = parse("test test test nonsense\n");
+ Assert.assertEquals(0, obj.size());
+ }
+
+ @Test
+ public void testTimestampPriority() throws java.text.ParseException {
+ long correctTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400")
+ .getTime();
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz");
+
+ for (JSONObject obj : parse(
+ "LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016 09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertEquals(new Date(correctTime), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
+ }
+ for (JSONObject obj : parse(
+ "2016-06-01T09:29:11.356-04:00 host LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016 09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertEquals(new Date(correctTime), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
+ }
+ for (JSONObject obj : parse(
+ "2016-05-01T09:29:11.356-04:00 host LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016 09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertEquals(new Date(correctTime), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(correctTime, obj.get(Fields.TIMESTAMP.getName()));
+ }
+ for (JSONObject obj : parse(
+ "LEEF:2.0|Lancope|StealthWatch|1.0|41|src=10.0.0.1\tdevTime=May 1 2016 09:29:11.356 -0400\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertNotNull(obj.get(Fields.TIMESTAMP.getName()));
+ }
+
+ }
+
+ private void runMissingYear(Calendar expected, Calendar input) {
+ SimpleDateFormat sdf = new SimpleDateFormat("MMM dd HH:mm:ss.SSS");
+ for (JSONObject obj : parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|\t|src=10.0.0.1\tdevTime="
+ + sdf.format(input.getTime()) +
+ "\tdevTimeFormat=MMM dd HH:mm:ss.SSS" +
+ "\tdst=2.1.2.2\tspt=1232")) {
+ Assert.assertEquals(expected.getTime(), new Date((long) obj.get(Fields.TIMESTAMP.getName())));
+ Assert.assertEquals(expected.getTimeInMillis(), obj.get(Fields.TIMESTAMP.getName()));
+ }
+ }
+
+ @Test
+ public void testMissingYearFromDate() throws java.text.ParseException {
+ Calendar current = Calendar.getInstance();
+ Calendar correct = Calendar.getInstance();
+
+ correct.setTimeInMillis(current.getTimeInMillis());
+
+ runMissingYear(correct, current);
+ }
+
+ @Test
+ public void testFourDayFutureBecomesPast() {
+ Calendar current = Calendar.getInstance();
+ Calendar correct = Calendar.getInstance();
+
+ current.add(Calendar.DAY_OF_MONTH, 5);
+ // correct.setTime(current.getTime());
+ correct.setTimeInMillis(current.getTimeInMillis());
+ correct.add(Calendar.YEAR, -1);
+
+ runMissingYear(correct, current);
+ }
+
+ /**
+ * Sample from https://docs.imperva.com/bundle/cloud-application-security/page/more/example-logs.htm#logEx2
+ */
+ @Test
+ public void testLEEF_CEFlikeSample() {
+ List<JSONObject> parse = parse("LEEF:0|Incapsula|SIEMintegration|0|SQL Injection| fileId=3412364560000000008 sourceServiceName=test56111115.incaptest.co siteid=1333546 suid=300656 requestClientApplication=Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0 popName=mia cs2=true cs2Label=Javascript Support cs3=true cs3Label=CO Support cs1=NA cs1Label=Cap Support cs4=936e64c2-bdd1-4719-9bd0-2d882a72f30d cs4Label=VID cs5=bab1712be85b00ab21d20bf0d7b5db82701f27f53fbac19a [...]
+ JSONObject obj = parse.get(0);
+ Assert.assertNotNull(obj);
+ Assert.assertEquals("3412364560000000008", obj.get("fileId"));
+ Assert.assertEquals("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0", obj.get("requestClientApplication"));
+ Assert.assertTrue(obj.containsKey("longitude"));
+ Assert.assertFalse(obj.containsKey("cs8"));
+ Assert.assertFalse(obj.containsKey("cs8Label"));
+ }
+
+ @Test
+ public void testLEEFParserSample() throws Exception {
+ runTest("sample", Resources.readLines(Resources.getResource(getClass(), "sample.leef"), UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "sample.schema"), UTF_8));
+ }
+
+ private void runTest(String name, List<String> lines, String schema) throws Exception {
+ runTest(name, lines, schema, "");
+ }
+
+ private void runTest(String name, List<String> lines, String schema, String targetJson) throws Exception {
+ for (String inputString : lines) {
+ JSONObject parsed = parse(inputString).get(0);
+ Assert.assertNotNull(parsed);
+ Assert.assertNotNull(parsed.get(Fields.TIMESTAMP.getName()));
+ Assert.assertTrue((long) parsed.get(Fields.TIMESTAMP.getName()) > 0);
+
+ JSONParser parser = new JSONParser();
+
+ Map<?, ?> json = null;
+ json = (Map<?, ?>) parser.parse(parsed.toJSONString());
+ Assert.assertEquals(true, validateJsonData(schema, json.toString()));
+ }
+ }
+
+ private void assertSimpleSample(List<JSONObject> parse) {
+ JSONObject obj = parse.get(0);
+ Assert.assertNotNull(obj);
+ Assert.assertTrue(obj.containsKey(Fields.SRC_ADDR.getName()));
+ Assert.assertEquals("192.0.2.0", obj.get(Fields.SRC_ADDR.getName()));
+ }
+
+ @Test
+ public void testLEEF_1_0_versionIncluded() {
+ List<JSONObject> parse = parse("LEEF:1.0|Microsoft|MSExchange|4.0 SP1|15345| src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
+ assertSimpleSample(parse);
+ }
+
+ @Test
+ public void testLEEF_2_0() {
+ List<JSONObject> parse = parse("LEEF:2.0|Vendor|Product|Version|EventID| src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
+ assertSimpleSample(parse);
+ }
+
+ @Test
+ public void testLEEF_2_0_delimiterSpecified() {
+ List<JSONObject> parse = parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^| src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black");
+ assertSimpleSample(parse);
+ }
+
+ @Test
+ public void testLEEF_2_0_delimiterUsedIncorrectly() {
+ List<JSONObject> parse = parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^| src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=joe.black");
+ assertFalse(parse.get(0).containsKey(Fields.DST_ADDR));
+ }
+
+ @Test
+ public void testLEEFMultiLine() {
+ List<JSONObject> parse = parse("LEEF:2.0|Vendor|Product|Version|EventID| src=192.0.2.0\tdst=172.50.123.1\tsev=5\tcat=anomaly\tsrcPort=81\tdstPort=21\tusrName=line1" +
+ "\nLEEF:2.0|Vendor|Product|Version|EventID| src=192.0.2.1\tdst=172.50.123.2\tsev=6\tcat=anomaly\tsrcPort=82\tdstPort=22\tusrName=line2");
+ assertSimpleSample(parse);
+ assertEquals(2, parse.size());
+ }
+
+
+ @Test
+ public void testLEEFcustomdevTimeFormat() {
+ String customFormat = "yyyy-MM-dd HH:mm:ss.SSS zzz";
+ Date customDate = new Date();
+ DateFormat customFormatter = new SimpleDateFormat(customFormat);
+
+ List<JSONObject> parse = parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^| src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black^devTime=" + customFormatter.format(customDate) + "^devTimeFormat=" + customFormat);
+ JSONObject obj = parse.get(0);
+ assertEquals(obj.get(Fields.TIMESTAMP.getName()), customDate.getTime());
+ }
+
+ @Test
+ public void testLEEFdevTimeWithNoCustomFormat() {
+ String standardFormat = "MMM dd yyyy HH:mm:ss.SSS zzz";
+ Date customDate = new Date();
+ long expected = customDate.getTime();
+ DateFormat customFormatter = new SimpleDateFormat(standardFormat);
+
+ List<JSONObject> parse = parse("LEEF:2.0|Lancope|StealthWatch|1.0|41|^| src=192.0.2.0^dst=172.50.123.1^sev=5^cat=anomaly^srcPort=81^dstPort=21^usrName=joe.black^devTime=" + customFormatter.format(customDate));
+ JSONObject obj = parse.get(0);
+ assertEquals(obj.get(Fields.TIMESTAMP.getName()), expected);
+ }
+
+ protected boolean validateJsonData(final String jsonSchema, final String jsonData) throws Exception {
+ final JsonNode d = JsonLoader.fromString(jsonData);
+ final JsonNode s = JsonLoader.fromString(jsonSchema);
+
+ final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+ JsonValidator v = factory.getValidator();
+
+ ProcessingReport report = v.validate(s, d);
+
+ return report.toString().contains("success");
+ }
+
+ private List<JSONObject> parse(String string) {
+ Optional<MessageParserResult<JSONObject>> parse = parser.parseOptionalResult(string.getBytes(UTF_8));
+ Assert.assertTrue(parse.isPresent());
+ return parse.get().getMessages();
+ }
+}
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/leef/sample.leef b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/leef/sample.leef
new file mode 100644
index 0000000..41af31c
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/leef/sample.leef
@@ -0,0 +1,4 @@
+LEEF:2.0|Trend Micro|Deep Security Manager|<DSA version>|192|cat=System name=Alert Ended desc=Alert: CPU Warning Threshold Exceeded\nSubject: 10.201.114.164\nSeverity: Warning sev=3 src=10.201.114.164 usrName=System msg=Alert: CPUWarning Threshold Exceeded\nSubject: 10.201.114.164\nSeverity:Warning TrendMicroDsTenant=Primary
+LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|2002779|cat=Integrity Monitor name=Microsoft Windows - System file modified desc=Microsoft Windows - System file modified sev=8 cn1=37 cn1Label=Host ID dvchost=www.example.com TrendMicroDsTenant=Primary TrendMicroDsTenantId=0 act=updated suser=admin
+LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|4000030|cat=Anti-Malware name=HEU_AEGIS_CRYPT desc=HEU_AEGIS_CRYPT sev=6 cn1=241 cn1Label=Host ID dvc=10.0.0.1 TrendMicroDsTags=FS TrendMicroDsTenant=Primary TrendMicroDsTenantId=0 filePath=C:\\Windows\\System32\\virus.exe act=Terminate msg=Realtime TrendMicroDsMalwareTarget=Multiple TrendMicroDsMalwareTargetType=File System TrendMicroDsFileMD5=1947A1BC0982C5871FA3768CD025453E#011 TrendMicroDsFileSHA1=5AD084DDCD8F80FBF2EE3F0E4F812E81 [...]
+LEEF:2.0|Trend Micro|Deep Security Agent|<DSA version>|5000000|cat=Web Reputation name=WebReputation desc=WebReputation sev=6 cn1=3 cn1Label=Host ID dvchost=exch01.example.com TrendMicroDsTenant=Primary TrendMicroDsTenantId=0 request=http://yw.olx5x9ny.org.it/HvuauRH/eighgSS.htm msg=Suspicious
\ No newline at end of file
diff --git a/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/leef/sample.schema b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/leef/sample.schema
new file mode 100644
index 0000000..4727998
--- /dev/null
+++ b/metron-platform/metron-parsing/metron-parsers/src/test/resources/org/apache/metron/parsers/leef/sample.schema
@@ -0,0 +1,27 @@
+{
+ "title": "LEEF Sample Schema",
+ "type": "object",
+ "properties": {
+ "original_string": {
+ "type": "string"
+ },
+ "timestamp": {
+ "type": "integer"
+ },
+ "DeviceVendor": {
+ "type": "string"
+ },
+ "DeviceProduct": {
+ "type": "string"
+ },
+ "DeviceVersion": {
+ "type": "string"
+ },
+ "DeviceEvent": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "original_string", "timestamp",
+ "DeviceVendor", "DeviceProduct", "DeviceVersion" ]
+}
\ No newline at end of file