You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ky...@apache.org on 2017/02/21 21:20:42 UTC
incubator-metron git commit: METRON-157 Create CEF Parser
(simonellistonball via kylerichardson) closes apache/incubator-metron#451
Repository: incubator-metron
Updated Branches:
refs/heads/master 246acff0f -> 9e15cb6e2
METRON-157 Create CEF Parser (simonellistonball via kylerichardson) closes apache/incubator-metron#451
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/9e15cb6e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/9e15cb6e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/9e15cb6e
Branch: refs/heads/master
Commit: 9e15cb6e24872620ec4bf3c183d15dd6292b153d
Parents: 246acff
Author: simonellistonball <si...@simonellistonball.com>
Authored: Tue Feb 21 15:50:51 2017 -0500
Committer: Kyle Richardson <ky...@apache.org>
Committed: Tue Feb 21 15:50:51 2017 -0500
----------------------------------------------------------------------
metron-platform/metron-parsers/pom.xml | 2 +-
.../apache/metron/parsers/cef/CEFParser.java | 274 ++++++++++++++++++
.../apache/metron/parsers/utils/DateUtils.java | 115 ++++++++
.../metron/parsers/cef/CEFParserTest.java | 277 +++++++++++++++++++
.../org/apache/metron/parsers/cef/adallom.cef | 1 +
.../apache/metron/parsers/cef/adallom.schema | 37 +++
.../org/apache/metron/parsers/cef/cyberark.cef | 1 +
.../org/apache/metron/parsers/cef/cyberark.json | 21 ++
.../apache/metron/parsers/cef/cyberark.schema | 38 +++
.../org/apache/metron/parsers/cef/waf.cef | 1 +
.../org/apache/metron/parsers/cef/waf.schema | 67 +++++
11 files changed, 833 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/pom.xml b/metron-platform/metron-parsers/pom.xml
index d8a77a0..3049a71 100644
--- a/metron-platform/metron-parsers/pom.xml
+++ b/metron-platform/metron-parsers/pom.xml
@@ -143,7 +143,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>${global_hbase_guava_version}</version>
+ <version>${global_guava_version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
new file mode 100644
index 0000000..a765dd8
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/cef/CEFParser.java
@@ -0,0 +1,274 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.cef;
+
+import java.nio.charset.Charset;
+import java.time.Clock;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.metron.parsers.BasicParser;
+import org.apache.metron.parsers.ParseException;
+import org.apache.metron.parsers.utils.DateUtils;
+import org.apache.metron.parsers.utils.SyslogUtils;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CEFParser extends BasicParser {
+ private static final long serialVersionUID = 1L;
+
+ protected static final Logger LOG = LoggerFactory.getLogger(CEFParser.class);
+ private static final String HEADER_CAPTURE_PATTERN = "[^\\|]*";
+ private static final String EXTENSION_CAPTURE_PATTERN = "(?<!\\\\)=";
+ private static final Charset UTF_8 = Charset.forName("UTF-8");
+
+ private Pattern p;
+ private Pattern pext;
+
+ public void init() {
+
+ // CEF Headers: Device Vendor|Device Product|Device Version|Device Event
+ // Class ID|Name|Severity
+
+ String syslogTime = "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\\b +(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?!<[0-9])(?:2[0123]|[01]?[0-9]):(?:[0-5][0-9])(?::(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?))(?![0-9])?";
+ String syslogTime5424 = "(?:\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:\\d{2}))";
+ String syslogPriority = "<(?:[0-9]+)>";
+ String syslogHost = "[a-z0-9\\.\\\\-_]+";
+
+ StringBuilder sb = new StringBuilder("(?<syslogTime>");
+ sb.append(syslogTime);
+ sb.append("|");
+ sb.append(syslogTime5424);
+ sb.append(")?");
+
+ sb.append("(?<syslogHost>");
+ sb.append(syslogHost);
+ sb.append(")?");
+
+ sb.append("(?<syslogPriority>");
+ sb.append(syslogPriority);
+ sb.append(")?");
+
+ sb.append(".*");
+
+ sb.append("CEF:0\\|");
+
+ headerBlock("DeviceVendor", sb);
+ sb.append("\\|");
+ headerBlock("DeviceProduct", sb);
+ sb.append("\\|");
+ headerBlock("DeviceVersion", sb);
+ sb.append("\\|");
+ headerBlock("DeviceEvent", sb);
+ sb.append("\\|");
+ headerBlock("Name", sb);
+ sb.append("\\|");
+ headerBlock("Severity", sb);
+ sb.append("\\|");
+
+ // extension capture:
+ sb.append("(?<extensions>.*)");
+ String pattern = sb.toString();
+
+ p = Pattern.compile(pattern);
+
+ // key finder for extensions
+ pext = Pattern.compile(EXTENSION_CAPTURE_PATTERN);
+ }
+
+ @SuppressWarnings("unchecked")
+ public List<JSONObject> parse(byte[] rawMessage) {
+ List<JSONObject> messages = new ArrayList<>();
+
+ String cefString = new String(rawMessage, UTF_8);
+
+ Matcher matcher = p.matcher(cefString);
+
+ while (matcher.find()) {
+ JSONObject obj = new JSONObject();
+ if (matcher.matches()) {
+ LOG.info(String.format("Found %d groups", matcher.groupCount()));
+ obj.put("DeviceVendor", matcher.group("DeviceVendor"));
+ obj.put("DeviceProduct", matcher.group("DeviceProduct"));
+ obj.put("DeviceVersion", matcher.group("DeviceVersion"));
+ obj.put("DeviceEvent", matcher.group("DeviceEvent"));
+ obj.put("Name", matcher.group("Name"));
+ obj.put("Severity", standardizeSeverity(matcher.group("Severity")));
+ }
+
+ String ext = matcher.group("extensions");
+ Matcher m = pext.matcher(ext);
+
+ int index = 0;
+ String key = null;
+ String value = null;
+ Map<String, String> labelMap = new HashMap<String, String>();
+
+ while (m.find()) {
+ if (key == null) {
+ key = ext.substring(index, m.start());
+ index = m.end();
+ if (!m.find()) {
+ break;
+ }
+ }
+ value = ext.substring(index, m.start());
+ index = m.end();
+ int v = value.lastIndexOf(" ");
+ if (v > 0) {
+ String temp = value.substring(0, v).trim();
+ if (key.endsWith("Label")) {
+ labelMap.put(key.substring(0, key.length() - 5), temp);
+ } else {
+ obj.put(key, temp);
+ }
+ key = value.substring(v).trim();
+ }
+ }
+ value = ext.substring(index);
+
+ // Build a map of Label extensions to apply later
+ if (key.endsWith("Label")) {
+ labelMap.put(key.substring(0, key.length() - 5), value);
+ } else {
+ obj.put(key, value);
+ }
+
+ // Apply the labels to custom fields
+ for (Entry<String, String> label : labelMap.entrySet()) {
+ mutate(obj, label.getKey(), label.getValue());
+ }
+
+ // Rename standard CEF fields to comply with Metron standards
+ obj = mutate(obj, "dst", "ip_dst_addr");
+ obj = mutate(obj, "dpt", "ip_dst_port");
+ obj = convertToInt(obj, "ip_dst_port");
+
+ obj = mutate(obj, "src", "ip_src_addr");
+ obj = mutate(obj, "spt", "ip_src_port");
+ obj = convertToInt(obj, "ip_src_port");
+
+ obj = mutate(obj, "act", "deviceAction");
+ // applicationProtocol
+ obj = mutate(obj, "app", "protocol");
+
+ obj.put("original_string", cefString);
+
+ // apply timestamp from message if present, using rt, syslog
+ // timestamp,
+ // default to current system time
+
+ if (obj.containsKey("rt")) {
+ String rt = (String) obj.get("rt");
+ try {
+ obj.put("timestamp", DateUtils.parseMultiformat(rt, DateUtils.DATE_FORMATS_CEF));
+ } catch (java.text.ParseException e) {
+ throw new IllegalStateException("rt field present in CEF but cannot be parsed", e);
+ }
+ } else {
+ String logTimestamp = matcher.group("syslogTime");
+ if (!(logTimestamp == null || logTimestamp.isEmpty())) {
+ try {
+ obj.put("timestamp", SyslogUtils.parseTimestampToEpochMillis(logTimestamp, Clock.systemUTC()));
+ } catch (ParseException e) {
+ throw new IllegalStateException("Cannot parse syslog timestamp", e);
+ }
+ } else {
+ obj.put("timestamp", System.currentTimeMillis());
+ }
+ }
+
+ // add the host
+ String host = matcher.group("syslogHost");
+ if (!(host == null || host.isEmpty())) {
+ obj.put("host", host);
+ }
+
+ messages.add(obj);
+ }
+ return messages;
+ }
+
+ @SuppressWarnings("unchecked")
+ private JSONObject convertToInt(JSONObject obj, String key) {
+ if (obj.containsKey(key)) {
+ obj.put(key, Integer.valueOf((String) obj.get(key)));
+ }
+ return obj;
+ }
+
+ private void headerBlock(String name, StringBuilder sb) {
+ sb.append("(?<").append(name).append(">").append(HEADER_CAPTURE_PATTERN).append(")");
+ }
+
+ /**
+ * Maps string based severity in CEF format to integer.
+ *
+ * The strings are mapped according to the CEF 23 specification, taking the
+ * integer value as the value of the range buckets rounded up
+ *
+ * The valid string values are: Unknown, Low, Medium, High, and Very-High.
+ * The valid integer values are: 0-3=Low, 4-6=Medium, 7- 8=High, and
+ * 9-10=Very-High.
+ *
+ * @param severity
+ * String or Integer
+ * @return Integer value mapped from the string
+ */
+ private Integer standardizeSeverity(String severity) {
+ if (severity.length() < 3) {
+ // should be a number
+ return Integer.valueOf(severity);
+ } else {
+ switch (severity) {
+ case "Low":
+ return 2;
+ case "Medium":
+ return 5;
+ case "High":
+ return 8;
+ case "Very-High":
+ return 10;
+ default:
+ return 0;
+ }
+ }
+ }
+
+ @Override
+ public void configure(Map<String, Object> config) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @SuppressWarnings("unchecked")
+ private JSONObject mutate(JSONObject json, String oldKey, String newKey) {
+ if (json.containsKey(oldKey)) {
+ json.put(newKey, json.remove(oldKey));
+ }
+ return json;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
new file mode 100644
index 0000000..888649a
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/utils/DateUtils.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.utils;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.TimeZone;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Various utilities for parsing and extracting dates
+ *
+ */
+public class DateUtils {
+
+ public static List<SimpleDateFormat> DATE_FORMATS_CEF = new ArrayList<SimpleDateFormat>() {
+ {
+ // as per CEF Spec
+ add(new SimpleDateFormat("MMM dd HH:mm:ss.SSS zzz"));
+ add(new SimpleDateFormat("MMM dd HH:mm:ss.SSS"));
+ add(new SimpleDateFormat("MMM dd HH:mm:ss zzz"));
+ add(new SimpleDateFormat("MMM dd HH:mm:ss"));
+ add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss.SSS zzz"));
+ add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss.SSS"));
+ add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss zzz"));
+ add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss"));
+ // found in the wild
+ add(new SimpleDateFormat("dd MMMM yyyy HH:mm:ss"));
+ }
+ };
+
+ public static List<SimpleDateFormat> DATE_FORMATS_SYSLOG = new ArrayList<SimpleDateFormat>() {
+ {
+ // As specified in https://tools.ietf.org/html/rfc5424
+ add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ"));
+
+ // common format per rsyslog defaults e.g. Mar 21 14:05:02
+ add(new SimpleDateFormat("MMM dd HH:mm:ss"));
+ add(new SimpleDateFormat("MMM dd yyyy HH:mm:ss"));
+
+ // additional formats found in the wild
+ add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"));
+ add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"));
+ add(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"));
+
+ }
+ };
+
+ Pattern NUMERIC = Pattern.compile("\\b\\d+\\b");
+
+ /**
+ * Parse the data according to a sequence of possible parse patterns.
+ *
+ * If the given date is entirely numeric, it is assumed to be a unix
+ * timestamp.
+ *
+ * If the year is not specified in the date string, use the current year.
+ * Assume that any date more than 4 days in the future is in the past as per
+ * SyslogUtils
+ *
+ * @param candidate
+ * The possible date.
+ * @param validPatterns
+ * A list of SimpleDateFormat instances to try parsing with.
+ * @return A java.util.Date based on the parse result
+ * @throws ParseException
+ */
+ public static long parseMultiformat(String candidate, List<SimpleDateFormat> validPatterns) throws ParseException {
+ if (StringUtils.isNumeric(candidate)) {
+ return Long.valueOf(candidate);
+ } else {
+ for (SimpleDateFormat pattern : validPatterns) {
+ try {
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(pattern.parse(candidate));
+ Calendar current = Calendar.getInstance();
+ if (cal.get(Calendar.YEAR) == 1970) {
+ cal.set(Calendar.YEAR, current.get(Calendar.YEAR));
+ }
+ current.add(Calendar.DAY_OF_MONTH, 4);
+ if (cal.after(current)) {
+ cal.add(Calendar.YEAR, -1);
+ }
+ return cal.getTimeInMillis();
+ } catch (ParseException e) {
+ continue;
+ }
+ }
+ throw new ParseException("Failed to parse any of the given date formats", 0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
new file mode 100644
index 0000000..88c0f0c
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
@@ -0,0 +1,277 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.metron.parsers.cef;
+
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.json.simple.parser.ParseException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.fge.jackson.JsonLoader;
+import com.github.fge.jsonschema.core.report.ProcessingReport;
+import com.github.fge.jsonschema.main.JsonSchemaFactory;
+import com.github.fge.jsonschema.main.JsonValidator;
+import com.google.common.io.Resources;
+
+import junit.framework.TestCase;
+
+public class CEFParserTest extends TestCase {
+
+ private static final Charset UTF_8 = Charset.forName("utf-8");
+ private CEFParser parser;
+
+ @Override
+ public void setUp() {
+ parser = new CEFParser();
+ parser.init();
+ }
+
+ @Test
+ public void testInvalid() {
+ List<JSONObject> obj = parse("test test test nonsense\n");
+ assertEquals(0, obj.size());
+ }
+
+ @Test
+ public void testEscaping() {
+ for (JSONObject obj : parse(
+ "Sep 19 08:26:10 host CEF:0|security|threatmanager|1.0|100|detected a \\ in packet|10|src=10.0.0.1 act=blocked a \\ dst=1.1.1.1")) {
+ assertEquals("10.0.0.1", obj.get("ip_src_addr"));
+ assertEquals("blocked a \\", obj.get("deviceAction"));
+ assertEquals("1.1.1.1", obj.get("ip_dst_addr"));
+ }
+ }
+
+ public void testBasicHeader() {
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
+ assertEquals("Security", obj.get("DeviceVendor"));
+ assertEquals("threatmanager", obj.get("DeviceProduct"));
+ assertEquals("1.0", obj.get("DeviceVersion"));
+ assertEquals("100", obj.get("DeviceEvent"));
+ assertEquals("worm successfully stopped", obj.get("Name"));
+ assertEquals(10, obj.get("Severity"));
+ }
+ }
+
+ public void testBasicExtensions() {
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
+ assertEquals("10.0.0.1", obj.get("ip_src_addr"));
+ assertEquals("2.1.2.2", obj.get("ip_dst_addr"));
+ assertEquals(1232, obj.get("ip_src_port"));
+ }
+ }
+
+ public void testCustomLabelWithSpace() {
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232 custom=Text with space customLabel=Label with space")) {
+ assertEquals(true, obj.containsKey("Label with space"));
+ assertEquals("Text with space", obj.get("Label with space"));
+ }
+ }
+
+ public void testTimestampPriority() throws java.text.ParseException {
+ long correctTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400")
+ .getTime();
+
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz");
+
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt=May 1 2016 09:29:11.356 -0400 dst=2.1.2.2 spt=1232")) {
+ assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp")));
+ assertEquals(correctTime, obj.get("timestamp"));
+ }
+ for (JSONObject obj : parse(
+ "2016-06-01T09:29:11.356-04:00 host CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt=May 1 2016 09:29:11.356 -0400 dst=2.1.2.2 spt=1232")) {
+ assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp")));
+ assertEquals(correctTime, obj.get("timestamp"));
+ }
+ for (JSONObject obj : parse(
+ "2016-05-01T09:29:11.356-04:00 host CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
+ assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp")));
+ assertEquals(correctTime, obj.get("timestamp"));
+ }
+ for (JSONObject obj : parse(
+ "CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2 spt=1232")) {
+ assertNotNull(obj.get("timestamp"));
+ }
+
+ }
+
+ public void testRtValueAsEpochTimestamp() throws java.text.ParseException {
+ long correctTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz").parse("2016-05-01T09:29:11.356-0400")
+ .getTime();
+ for (JSONObject obj : parse("CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt="
+ + String.valueOf(correctTime) + " dst=2.1.2.2 spt=1232")) {
+ assertEquals(new Date(correctTime), new Date((long) obj.get("timestamp")));
+ assertEquals(correctTime, obj.get("timestamp"));
+ }
+ }
+
+ private void runMissingYear(Calendar expected, Calendar input) {
+ SimpleDateFormat sdf = new SimpleDateFormat("MMM dd HH:mm:ss.SSS");
+ for (JSONObject obj : parse("CEF:0|Security|threatmanager|1.0|100|worm successfully stopped|10|src=10.0.0.1 rt="
+ + sdf.format(input.getTime()) + " dst=2.1.2.2 spt=1232")) {
+ assertEquals(expected.getTimeInMillis(), obj.get("timestamp"));
+ assertEquals(expected.getTime(), new Date((long) obj.get("timestamp")));
+ }
+ }
+
+ public void testMissingYearFromDate() throws java.text.ParseException {
+ Calendar current = Calendar.getInstance();
+ Calendar correct = Calendar.getInstance();
+
+ correct.setTimeInMillis(current.getTimeInMillis());
+
+ runMissingYear(correct, current);
+ }
+
+ public void testFourDayFutureBecomesPast() {
+ Calendar current = Calendar.getInstance();
+ Calendar correct = Calendar.getInstance();
+
+ current.add(Calendar.DAY_OF_MONTH, 5);
+ // correct.setTime(current.getTime());
+ correct.setTimeInMillis(current.getTimeInMillis());
+ correct.add(Calendar.YEAR, -1);
+
+ runMissingYear(correct, current);
+ }
+
+ public void testCEFParserAdallom() throws Exception {
+ runTest("adallom", Resources.readLines(Resources.getResource(getClass(), "adallom.cef"), UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "adallom.schema"), UTF_8));
+ }
+
+ public void testCEFParserCyberArk() throws Exception {
+ runTest("cyberark", Resources.readLines(Resources.getResource(getClass(), "cyberark.cef"), UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "cyberark.schema"), UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "cyberark.json"), UTF_8));
+ }
+
+ public void testCEFParserWAF() throws Exception {
+ URL waf_url = Resources.getResource(getClass(), "waf.cef");
+ runTest("waf", Resources.readLines(waf_url, UTF_8),
+ Resources.toString(Resources.getResource(getClass(), "waf.schema"), UTF_8));
+ }
+
+ private void runTest(String name, List<String> lines, String schema) throws Exception {
+ runTest(name, lines, schema, "");
+ }
+
+ private void runTest(String name, List<String> lines, String schema, String targetJson) throws Exception {
+ for (String inputString : lines) {
+ JSONObject parsed = parse(inputString).get(0);
+ assertNotNull(parsed);
+ assertNotNull(parsed.get("timestamp"));
+ assertTrue((long) parsed.get("timestamp") > 0);
+
+ System.out.println(parsed);
+ JSONParser parser = new JSONParser();
+
+ Map<?, ?> json = null;
+ try {
+ json = (Map<?, ?>) parser.parse(parsed.toJSONString());
+ Assert.assertEquals(true, validateJsonData(schema, json.toString()));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ }
+
+ // test against an explicit json example
+ if (!targetJson.isEmpty()) {
+
+ }
+ }
+ }
+
+ /**
+ * Additional Sample from NiFi test Suite
+ * (https://github.com/apache/nifi/blob/rel/nifi-1.1.1/nifi-nar-bundles/nifi
+ * -standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/
+ * processors/standard/TestParseCEF.java)
+ */
+ private final static String sample = "CEF:0|TestVendor|TestProduct|TestVersion|TestEventClassID|TestName|Low|" +
+ // TimeStamp, String and Long
+ "rt=Feb 09 2015 00:27:43 UTC cn3Label=Test Long cn3=9223372036854775807 " +
+ // FloatPoint and MacAddress
+ "cfp1=1.234 cfp1Label=Test FP Number smac=00:00:0c:07:ac:00 " +
+ // IPv6 and String
+ "c6a3=2001:cdba::3257:9652 c6a3Label=Test IPv6 cs1Label=Test String cs1=test test test chocolate " +
+ // IPv4
+ "destinationTranslatedAddress=123.123.123.123 " +
+ // Date without TZ
+ "deviceCustomDate1=Feb 06 2015 13:27:43 " +
+ // Integer and IP Address (from v4)
+ "dpt=1234 agt=123.123.0.124 dlat=40.366633 " +
+ // A JSON object inside one of CEF's custom Strings
+ "cs2Label=JSON payload "
+ + "cs2={\"test_test_test\": \"chocolate!\", \"what?!?\": \"Simple! test test test chocolate!\"}";
+
+ @Test
+ public void testSuccessfulWhenCEFContainsJSON() throws JsonProcessingException, IOException {
+ List<JSONObject> parse = parse(sample);
+ JSONObject obj = parse.get(0);
+
+ assertEquals("TestVendor", obj.get("DeviceVendor"));
+ assertEquals(1423441663000L, obj.get("timestamp"));
+ assertEquals("9223372036854775807", obj.get("Test Long"));
+ assertEquals(obj.get("Test FP Number"), String.valueOf(1.234F));
+ assertEquals("00:00:0c:07:ac:00", obj.get("smac"));
+ assertEquals("2001:cdba::3257:9652", obj.get("Test IPv6"));
+ assertEquals("test test test chocolate", obj.get("Test String"));
+ assertEquals("123.123.123.123", obj.get("destinationTranslatedAddress"));
+
+ JsonNode inner = new ObjectMapper().readTree((String) obj.get("JSON payload"));
+ Assert.assertEquals("chocolate!", inner.get("test_test_test").asText());
+ }
+
+ protected boolean validateJsonData(final String jsonSchema, final String jsonData) throws Exception {
+ final JsonNode d = JsonLoader.fromString(jsonData);
+ final JsonNode s = JsonLoader.fromString(jsonSchema);
+
+ final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+ JsonValidator v = factory.getValidator();
+
+ ProcessingReport report = v.validate(s, d);
+ System.out.println(report);
+
+ return report.toString().contains("success");
+ }
+
+ private List<JSONObject> parse(String string) {
+ List<JSONObject> parse = parser.parse(string.getBytes(Charset.forName("utf-8")));
+ assertNotNull(parse);
+ return parse;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
new file mode 100644
index 0000000..a35f354
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.cef
@@ -0,0 +1 @@
+2016-04-01T09:29:11.356-0400 CEF:0|Adallom|Adallom|1.0|56fe779ee4b0459f4e9a484a|ALERT_CABINET_EVENT_MATCH_AUDIT|0|msg=Activity policy 'User download/view file' was triggered by 'person@example.com' suser=auser@example.com start=1459517280810 end=1459517280810 audits=["AVPR-4oIPeFmuZ3CKKrg","AVPR-wx80cd9PUpAu2aj","AVPR-6XGPeFmuZ3CKKvx","AVPSALn_qE4Kgs_8_yK9","AVPSASW3gw_f3aEvgEmi"] services=["APPID_SXC"] users=["another@example.com"] cs6=https://abcd-remote.console.arc.com/#/alerts/56fe779ee4b0459f4e9a484a cs6Label=consoleUrl
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
new file mode 100644
index 0000000..a91cce0
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/adallom.schema
@@ -0,0 +1,37 @@
+{
+ "title": "Adallom Schema",
+ "type": "object",
+ "properties": {
+ "original_string": {
+ "type": "string"
+ },
+ "timestamp": {
+ "type": "integer"
+ },
+ "DeviceVendor": {
+ "type": "string"
+ },
+ "DeviceProduct": {
+ "type": "string"
+ },
+ "DeviceVersion": {
+ "type": "string"
+ },
+ "DeviceEvent": {
+ "type": "string"
+ },
+ "Name": {
+ "type": "string"
+ },
+ "Severity": {
+ "type": "integer"
+ },
+ "consoleUrl": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "original_string", "timestamp",
+ "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity",
+ "consoleUrl"]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
new file mode 100644
index 0000000..9d4fe6f
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.cef
@@ -0,0 +1 @@
+Mar 21 14:05:02 HHHPVATN1 CEF:0|Cyber-Ark|Vault|7.20.0091|295|Retrieve password|5|act=Retrieve password suser=spilgrim fname=Root\ABC phobos3 - COMP dvc=120.99.70.3 shost=10.44.134.78 dhost= duser= externalId= app= reason= cs1Label="Affected User Name" cs1= cs2Label="Safe Name" cs2=Security Vulnerability Mgmt cs3Label="Device Type" cs3= cs4Label="Database" cs4= cs5Label="Other info" cs5=101.198.70.93 cn1Label="Request Id" cn1= cn2Label="Ticket Id" cn2=Needed to verify config files being pulled msg=Needed to verify config files being pulled
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
new file mode 100644
index 0000000..e900a9a
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.json
@@ -0,0 +1,21 @@
+{
+ "\"Other info\"": "101.198.70.93",
+ "\"Safe Name\"": "Security Vulnerability Mgmt",
+ "\"Ticket Id\"": "Needed to verify config files being pulled ",
+ "deviceAction": "Retrieve password",
+ "deviceAddress": "120.99.70.3",
+ "device_product": "Vault",
+ "device_vendor": "Cyber-Ark",
+ "device_version": "7.20.0091",
+ "event_class_id": "295",
+ "event_name": "Retrieve password",
+ "fileName": "Root\\ABC phobos3 - COMP",
+ "header": "Mar 21 14:05:02 HHHPVATN1 CEF:0",
+ "message": "Needed to verify config files being pulled",
+ "original_string": "Mar 21 14:05:02 HHHPVATN1 CEF:0|Cyber-Ark|Vault|7.20.0091|295|Retrieve password|5|act=Retrieve password suser=spilgrim fname=Root\\ABC phobos3 - COMP dvc=120.99.70.3 shost=10.44.134.78 dhost= duser= externalId= app= reason= cs1Label=\"Affected User Name\" cs1= cs2Label=\"Safe Name\" cs2=Security Vulnerability Mgmt cs3Label=\"Device Type\" cs3= cs4Label=\"Database\" cs4= cs5Label=\"Other info\" cs5=101.198.70.93 cn1Label=\"Request Id\" cn1= cn2Label=\"Ticket Id\" cn2=Needed to verify config files being pulled msg=Needed to verify config files being pulled",
+ "severity": "5",
+ "source.type": "cyberark",
+ "src_hostname": "10.44.134.78",
+ "src_username": "spilgrim",
+ "timestamp": 1458569102000
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
new file mode 100644
index 0000000..5bd1021
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/cyberark.schema
@@ -0,0 +1,38 @@
+{
+ "title": "CyberArk Schema",
+ "type": "object",
+ "properties": {
+ "ip_src_addr": {
+ "type": "string"
+ },
+ "ip_dst_addr": {
+ "type": "string"
+ },
+ "original_string": {
+ "type": "string"
+ },
+ "timestamp": {
+ "type": "integer"
+ },
+ "DeviceVendor": {
+ "type": "string"
+ },
+ "DeviceProduct": {
+ "type": "string"
+ },
+ "DeviceVersion": {
+ "type": "string"
+ },
+ "DeviceEvent": {
+ "type": "string"
+ },
+ "Name": {
+ "type": "string"
+ },
+ "Severity": {
+ "type": "integer"
+ }
+ },
+ "required": ["original_string", "timestamp",
+ "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity"]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef
new file mode 100644
index 0000000..86e1d6b
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.cef
@@ -0,0 +1 @@
+<14>CEF:0|Imperva Inc.|SecureSphere|10.0.0.4_16|ABC - Secure Login.vm Page Rate Limit UK - Source IP||High|act=alert dst=17.43.200.42 dpt=88 duser=${Alert.username} src=10.31.45.69 spt=34435 proto=TCP rt=31 March 2016 13:04:55 cat=Alert cs1= cs1Label=Policy cs2=ABC-Secure cs2Label=ServerGroup cs3=servers_svc cs3Label=ServiceName cs4=server_app cs4Label=ApplicationName cs5=QA cs5Label=Description
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/9e15cb6e/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema
new file mode 100644
index 0000000..b38485c
--- /dev/null
+++ b/metron-platform/metron-parsers/src/test/resources/org/apache/metron/parsers/cef/waf.schema
@@ -0,0 +1,67 @@
+{
+ "title": "WAF CEF Schema",
+ "type": "object",
+ "properties": {
+ "ip_src_addr": {
+ "type": "string"
+ },
+ "ip_src_port": {
+ "type": "integer"
+ },
+ "ip_dst_addr": {
+ "type": "string"
+ },
+ "ip_dst_port": {
+ "type": "integer"
+ },
+ "original_string": {
+ "type": "string"
+ },
+ "@version": {
+ "type": "string"
+ },
+ "timestamp": {
+ "type": "integer"
+ },
+ "type": {
+ "type": "string"
+ },
+ "DeviceVendor": {
+ "type": "string"
+ },
+ "DeviceProduct": {
+ "type": "string"
+ },
+ "DeviceVersion": {
+ "type": "string"
+ },
+ "DeviceEvent": {
+ "type": "string"
+ },
+ "Name": {
+ "type": "string"
+ },
+ "Severity": {
+ "type": "integer"
+ },
+ "cat": {
+ "type": "string"
+ },
+ "ServerGroup": {
+ "type": "string"
+ },
+ "ServiceName": {
+ "type": "string"
+ },
+ "ApplicationName": {
+ "type": "string"
+ },
+ "Description": {
+ "type": "string"
+ }
+ },
+ "required": ["ip_src_addr", "ip_dst_addr", "ip_src_port", "ip_dst_port", "original_string", "timestamp",
+ "DeviceVendor", "DeviceProduct", "DeviceVersion", "Name", "Severity",
+ "cat",
+ "ServerGroup", "ServiceName", "ApplicationName", "Description"]
+}
\ No newline at end of file