You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by mm...@apache.org on 2016/10/11 18:00:28 UTC
incubator-metron git commit: METRON-488: Snort should use a proper
CSV implementation (cestella via mmiklavc) closes apache/incubator-metron#297
Repository: incubator-metron
Updated Branches:
refs/heads/master 6f5c84ee2 -> 609ea40c2
METRON-488: Snort should use a proper CSV implementation (cestella via mmiklavc) closes apache/incubator-metron#297
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/609ea40c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/609ea40c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/609ea40c
Branch: refs/heads/master
Commit: 609ea40c2e8ab3b35b645f4238d2f66afc120fb5
Parents: 6f5c84e
Author: cstella <ce...@gmail.com>
Authored: Tue Oct 11 13:59:45 2016 -0400
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Oct 11 13:59:45 2016 -0400
----------------------------------------------------------------------
.../main/sample/data/snort/parsed/SnortParsed | 6 +--
.../metron/parsers/snort/BasicSnortParser.java | 39 +++++++++++++++-----
.../apache/metron/parsers/SnortParserTest.java | 6 ++-
3 files changed, 37 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/609ea40c/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed b/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed
index 318b158..e2894b3 100644
--- a/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed
+++ b/metron-platform/metron-integration-test/src/main/sample/data/snort/parsed/SnortParsed
@@ -1,3 +1,3 @@
-{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"22","ethsrc":"52:54:00:12:35:02","tcpseq":"0x9AFF3D7","dgmlen":"64","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0xC8761D52","original_string":"01\/27-16:01:04.877970 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,","icmpcode":"","tos":"0","id":"59677","timestamp":1453932941970,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"10.0.2.2","ttl":"64","source.type":"snort","ethlen":"0x4E","iplen":"65536","icmptype":"","protocol":"TCP","ip_src_port":"56642","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
-{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB45F7A","dgmlen":"96","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.612494 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,","icmpcode":"","tos":"0","id":"16785","timestamp":1456178820494,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"96.44.142.5","ttl":"64","source.type":"snort","ethlen":"0x6E","iplen":"98304","icmptype":"","protocol":"TCP","ip_src_port":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
-{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB508F2","dgmlen":"152","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.616775 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,","icmpcode":"","tos":"0","id":"16824","timestamp":1456178824775,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"96.44.142.5","ttl":"64","source.type":"snort","ethlen":"0xA6","iplen":"155648","icmptype":"","protocol":"TCP","ip_src_port":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
+{"msg":"Consecutive TCP small segments exceeding threshold","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"22","ethsrc":"52:54:00:12:35:02","tcpseq":"0x9AFF3D7","dgmlen":"64","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0xC8761D52","original_string":"01\/27-16:01:04.877970 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,","icmpcode":"","tos":"0","id":"59677","timestamp":1453932941970,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"10.0.2.2","ttl":"64","source.type":"snort","ethlen":"0x4E","iplen":"65536","icmptype":"","protocol":"TCP","ip_src_port":"56642","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
+{"msg":"Consecutive TCP small segments exceeding threshold","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB45F7A","dgmlen":"96","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.612494 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,","icmpcode":"","tos":"0","id":"16785","timestamp":1456178820494,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"96.44.142.5","ttl":"64","source.type":"snort","ethlen":"0x6E","iplen":"98304","icmptype":"","protocol":"TCP","ip_src_port":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
+{"msg":"Consecutive TCP small segments exceeding threshold","sig_rev":"1","ip_dst_addr":"10.0.2.15","ip_dst_port":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB508F2","dgmlen":"152","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.616775 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,","icmpcode":"","tos":"0","id":"16824","timestamp":1456178824775,"ethdst":"08:00:27:7F:93:2D","ip_src_addr":"96.44.142.5","ttl":"64","source.type":"snort","ethlen":"0xA6","iplen":"155648","icmptype":"","protocol":"TCP","ip_src_port":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/609ea40c/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
index f295d4c..e50926c 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
@@ -17,7 +17,10 @@
*/
package org.apache.metron.parsers.snort;
+import com.google.common.collect.Lists;
+import jdk.nashorn.internal.runtime.arrays.ArrayIndex;
import org.apache.metron.common.Constants;
+import org.apache.metron.common.csv.CSVConverter;
import org.apache.metron.parsers.BasicParser;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
@@ -72,14 +75,26 @@ public class BasicSnortParser extends BasicParser {
*/
private String recordDelimiter = ",";
+ private transient CSVConverter converter;
+
+ public BasicSnortParser() {
+
+ }
+
@Override
public void configure(Map<String, Object> parserConfig) {
-
+ init();
}
@Override
public void init() {
-
+ if(converter == null) {
+ converter = new CSVConverter();
+ Map<String, Object> config = new HashMap<>();
+ config.put(CSVConverter.SEPARATOR_KEY, recordDelimiter);
+ config.put(CSVConverter.COLUMNS_KEY, Lists.newArrayList(fieldNames));
+ converter.initialize(config);
+ }
}
@Override
@@ -90,18 +105,24 @@ public class BasicSnortParser extends BasicParser {
try {
// snort alerts expected as csv records
String csvMessage = new String(rawMessage, "UTF-8");
- String[] records = csvMessage.split(recordDelimiter, -1);
+ Map<String, String> records = null;
+ try {
+ records = converter.toMap(csvMessage);
+ }
+ catch(ArrayIndexOutOfBoundsException aioob) {
+ throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " in " + csvMessage);
+ }
// validate the number of fields
- if (records.length != fieldNames.length) {
- throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.length);
+ if (records.size() != fieldNames.length) {
+ throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.size());
}
long timestamp = 0L;
// build the json record from each field
- for (int i=0; i<records.length; i++) {
+ for (Map.Entry<String, String> kv : records.entrySet()) {
- String field = fieldNames[i];
- String record = records[i];
+ String field = kv.getKey();
+ String record = kv.getValue();
if("timestamp".equals(field)) {
@@ -119,7 +140,7 @@ public class BasicSnortParser extends BasicParser {
jsonMessage.put("is_alert", "true");
messages.add(jsonMessage);
} catch (Exception e) {
- String message = "Unable to parse message: " + rawMessage;
+ String message = "Unable to parse message: " + (rawMessage == null?"null" : new String(rawMessage));
_LOG.error(message, e);
throw new IllegalStateException(message, e);
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/609ea40c/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
index 6d777aa..8114e83 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/SnortParserTest.java
@@ -27,7 +27,7 @@ import java.util.Map;
public class SnortParserTest {
/**
- 01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,
+ 01/27-16:01:04.877970 ,129,12,1,"Consecutive TCP small segments, exceeding threshold",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,
**/
@Multiline
public static String goodMessage;
@@ -36,8 +36,9 @@ public class SnortParserTest {
@Test
public void testGoodMessage() {
BasicSnortParser parser = new BasicSnortParser();
+ parser.init();
Map out = parser.parse(goodMessage.getBytes()).get(0);
- Assert.assertEquals(out.get("msg"),"\"Consecutive TCP small segments exceeding threshold\"");
+ Assert.assertEquals(out.get("msg"),"Consecutive TCP small segments, exceeding threshold");
Assert.assertEquals(out.get("sig_rev"), "1");
Assert.assertEquals(out.get("ip_dst_addr"), "10.0.2.15");
Assert.assertEquals(out.get("ip_dst_port"), "22");
@@ -69,6 +70,7 @@ public class SnortParserTest {
@Test(expected=IllegalStateException.class)
public void testBadMessage() {
BasicSnortParser parser = new BasicSnortParser();
+ parser.init();
parser.parse("foo bar".getBytes());
}
}