You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/06/16 18:52:46 UTC
incubator-metron git commit: METRON-230: Bro parser should throw
exception. This closes apache/incubator-metron#154
Repository: incubator-metron
Updated Branches:
refs/heads/master 0a3da362e -> 869096693
METRON-230: Bro parser should throw exception. This closes apache/incubator-metron#154
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/86909669
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/86909669
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/86909669
Branch: refs/heads/master
Commit: 869096693fb86631fb4caf4296e169d4527ce0da
Parents: 0a3da36
Author: cstella <ce...@gmail.com>
Authored: Thu Jun 16 14:52:38 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Thu Jun 16 14:52:38 2016 -0400
----------------------------------------------------------------------
.../metron/parsers/bro/BasicBroParser.java | 229 +++++++++----------
.../metron/parsers/bro/BasicBroParserTest.java | 18 ++
2 files changed, 132 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86909669/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
index 4052e86..76e4956 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
@@ -32,135 +32,134 @@ import java.util.Map;
@SuppressWarnings("serial")
public class BasicBroParser extends BasicParser {
- protected static final Logger _LOG = LoggerFactory
- .getLogger(BasicBroParser.class);
- private JSONCleaner cleaner = new JSONCleaner();
+ protected static final Logger _LOG = LoggerFactory
+ .getLogger(BasicBroParser.class);
+ private JSONCleaner cleaner = new JSONCleaner();
- @Override
- public void configure(Map<String, Object> parserConfig) {
+ @Override
+ public void configure(Map<String, Object> parserConfig) {
- }
+ }
- @Override
- public void init() {
+ @Override
+ public void init() {
- }
+ }
- @SuppressWarnings("unchecked")
- public List<JSONObject> parse(byte[] msg) {
+ @SuppressWarnings("unchecked")
+ public List<JSONObject> parse(byte[] msg) {
- _LOG.trace("[Metron] Starting to parse incoming message");
+ _LOG.trace("[Metron] Starting to parse incoming message");
- String rawMessage = null;
- List<JSONObject> messages = new ArrayList<>();
- try {
- rawMessage = new String(msg, "UTF-8");
- _LOG.trace("[Metron] Received message: " + rawMessage);
-
- JSONObject cleanedMessage = cleaner.clean(rawMessage);
- _LOG.debug("[Metron] Cleaned message: " + cleanedMessage);
-
- if (cleanedMessage == null || cleanedMessage.isEmpty()) {
- throw new Exception("Unable to clean message: " + rawMessage);
- }
-
- String key;
- JSONObject payload;
- if (cleanedMessage.containsKey("type")) {
- key = cleanedMessage.get("type").toString();
- payload = cleanedMessage;
- } else {
- key = cleanedMessage.keySet().iterator().next().toString();
-
- if (key == null) {
- throw new Exception("Unable to retrieve key for message: "
- + rawMessage);
- }
-
- payload = (JSONObject) cleanedMessage.get(key);
- }
-
- if (payload == null) {
- throw new Exception("Unable to retrieve payload for message: "
- + rawMessage);
- }
-
- String originalString = key.toUpperCase() + " |";
- for (Object k : payload.keySet()) {
- String value = payload.get(k).toString();
- originalString += " " + k.toString() + ":" + value;
- }
- payload.put("original_string", originalString);
-
- replaceKey(payload, Constants.Fields.TIMESTAMP.getName(), new String[]{ "ts" });
-
- long timestamp = 0L;
- if (payload.containsKey(Constants.Fields.TIMESTAMP.getName())) {
- try {
- String broTimestamp = payload.get(Constants.Fields.TIMESTAMP.getName()).toString();
- String convertedTimestamp = broTimestamp.replace(".","");
- convertedTimestamp = convertedTimestamp.substring(0,13);
- timestamp = Long.parseLong(convertedTimestamp);
- payload.put(Constants.Fields.TIMESTAMP.getName(), timestamp);
- payload.put("bro_timestamp",broTimestamp);
- _LOG.trace(String.format("[Metron] new bro record - timestamp : %s", payload.get(Constants.Fields.TIMESTAMP.getName())));
- } catch (NumberFormatException nfe) {
- _LOG.error(String.format("[Metron] timestamp is invalid: %s", payload.get("timestamp")));
- payload.put(Constants.Fields.TIMESTAMP.getName(), 0);
- }
- }
-
- boolean ipSrcReplaced = replaceKey(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{"source_ip", "id.orig_h"});
- if (!ipSrcReplaced) {
- replaceKeyArray(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{ "tx_hosts" });
- }
-
- boolean ipDstReplaced = replaceKey(payload, Constants.Fields.DST_ADDR.getName(), new String[]{"dest_ip", "id.resp_h"});
- if (!ipDstReplaced) {
- replaceKeyArray(payload, Constants.Fields.DST_ADDR.getName(), new String[]{ "rx_hosts" });
- }
-
- replaceKey(payload, Constants.Fields.SRC_PORT.getName(), new String[]{"source_port", "id.orig_p"});
- replaceKey(payload, Constants.Fields.DST_PORT.getName(), new String[]{"dest_port", "id.resp_p"});
-
- payload.put(Constants.Fields.PROTOCOL.getName(), key);
- _LOG.debug("[Metron] Returning parsed message: " + payload);
- messages.add(payload);
- return messages;
-
- } catch (Exception e) {
-
- _LOG.error("Unable to Parse Message: " + rawMessage);
- e.printStackTrace();
- return null;
+ String rawMessage = null;
+ List<JSONObject> messages = new ArrayList<>();
+ try {
+ rawMessage = new String(msg, "UTF-8");
+ _LOG.trace("[Metron] Received message: " + rawMessage);
+
+ JSONObject cleanedMessage = cleaner.clean(rawMessage);
+ _LOG.debug("[Metron] Cleaned message: " + cleanedMessage);
+
+ if (cleanedMessage == null || cleanedMessage.isEmpty()) {
+ throw new Exception("Unable to clean message: " + rawMessage);
+ }
+
+ String key;
+ JSONObject payload;
+ if (cleanedMessage.containsKey("type")) {
+ key = cleanedMessage.get("type").toString();
+ payload = cleanedMessage;
+ } else {
+ key = cleanedMessage.keySet().iterator().next().toString();
+
+ if (key == null) {
+ throw new Exception("Unable to retrieve key for message: "
+ + rawMessage);
}
- }
+ payload = (JSONObject) cleanedMessage.get(key);
+ }
+
+ if (payload == null) {
+ throw new Exception("Unable to retrieve payload for message: "
+ + rawMessage);
+ }
+
+ String originalString = key.toUpperCase() + " |";
+ for (Object k : payload.keySet()) {
+ String value = payload.get(k).toString();
+ originalString += " " + k.toString() + ":" + value;
+ }
+ payload.put("original_string", originalString);
+
+ replaceKey(payload, Constants.Fields.TIMESTAMP.getName(), new String[]{ "ts" });
- private boolean replaceKey(JSONObject payload, String toKey, String[] fromKeys) {
- for (String fromKey : fromKeys) {
- if (payload.containsKey(fromKey)) {
- Object value = payload.remove(fromKey);
- payload.put(toKey, value);
- _LOG.trace(String.format("[Metron] Added %s to %s", toKey, payload));
- return true;
- }
+ long timestamp = 0L;
+ if (payload.containsKey(Constants.Fields.TIMESTAMP.getName())) {
+ try {
+ String broTimestamp = payload.get(Constants.Fields.TIMESTAMP.getName()).toString();
+ String convertedTimestamp = broTimestamp.replace(".","");
+ convertedTimestamp = convertedTimestamp.substring(0,13);
+ timestamp = Long.parseLong(convertedTimestamp);
+ payload.put(Constants.Fields.TIMESTAMP.getName(), timestamp);
+ payload.put("bro_timestamp",broTimestamp);
+ _LOG.trace(String.format("[Metron] new bro record - timestamp : %s", payload.get(Constants.Fields.TIMESTAMP.getName())));
+ } catch (NumberFormatException nfe) {
+ _LOG.error(String.format("[Metron] timestamp is invalid: %s", payload.get("timestamp")));
+ payload.put(Constants.Fields.TIMESTAMP.getName(), 0);
}
- return false;
+ }
+
+ boolean ipSrcReplaced = replaceKey(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{"source_ip", "id.orig_h"});
+ if (!ipSrcReplaced) {
+ replaceKeyArray(payload, Constants.Fields.SRC_ADDR.getName(), new String[]{ "tx_hosts" });
+ }
+
+ boolean ipDstReplaced = replaceKey(payload, Constants.Fields.DST_ADDR.getName(), new String[]{"dest_ip", "id.resp_h"});
+ if (!ipDstReplaced) {
+ replaceKeyArray(payload, Constants.Fields.DST_ADDR.getName(), new String[]{ "rx_hosts" });
+ }
+
+ replaceKey(payload, Constants.Fields.SRC_PORT.getName(), new String[]{"source_port", "id.orig_p"});
+ replaceKey(payload, Constants.Fields.DST_PORT.getName(), new String[]{"dest_port", "id.resp_p"});
+
+ payload.put(Constants.Fields.PROTOCOL.getName(), key);
+ _LOG.debug("[Metron] Returning parsed message: " + payload);
+ messages.add(payload);
+ return messages;
+
+ } catch (Exception e) {
+ String message = "Unable to parse Message: " + rawMessage;
+ _LOG.error(message, e);
+ throw new IllegalStateException(message, e);
}
- private boolean replaceKeyArray(JSONObject payload, String toKey, String[] fromKeys) {
- for (String fromKey : fromKeys) {
- if (payload.containsKey(fromKey)) {
- JSONArray value = (JSONArray) payload.remove(fromKey);
- if (value != null && !value.isEmpty()) {
- payload.put(toKey, value.get(0));
- _LOG.trace(String.format("[Metron] Added %s to %s", toKey, payload));
- return true;
- }
- }
+ }
+
+ private boolean replaceKey(JSONObject payload, String toKey, String[] fromKeys) {
+ for (String fromKey : fromKeys) {
+ if (payload.containsKey(fromKey)) {
+ Object value = payload.remove(fromKey);
+ payload.put(toKey, value);
+ _LOG.trace(String.format("[Metron] Added %s to %s", toKey, payload));
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean replaceKeyArray(JSONObject payload, String toKey, String[] fromKeys) {
+ for (String fromKey : fromKeys) {
+ if (payload.containsKey(fromKey)) {
+ JSONArray value = (JSONArray) payload.remove(fromKey);
+ if (value != null && !value.isEmpty()) {
+ payload.put(toKey, value.get(0));
+ _LOG.trace(String.format("[Metron] Added %s to %s", toKey, payload));
+ return true;
}
- return false;
+ }
}
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/86909669/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
index 3ed1b2c..1f4f9ab 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/bro/BasicBroParserTest.java
@@ -160,4 +160,22 @@ public class BasicBroParserTest extends TestCase {
Assert.assertEquals(broJson.get("ip_src_addr").toString(), rawJson.get("id.orig_h").toString());
Assert.assertTrue(broJson.get("original_string").toString().startsWith("HTTP"));
}
+
+ public void testBadMessage() throws ParseException{
+ try {
+ broParser.parse("{ \"foo\" : \"bar\"}".getBytes());
+ Assert.fail("Should have marked this as a bad message.");
+ }
+ catch(IllegalStateException ise) {
+
+ }
+ //non json
+ try {
+ broParser.parse("foo bar".getBytes());
+ Assert.fail("Should have marked this as a bad message.");
+ }
+ catch(IllegalStateException ise) {
+
+ }
+ }
}