You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2016/05/02 19:26:44 UTC
[3/3] incubator-metron git commit: METRON-122 Create generic unit
test framework for testing grok statements (merrimanr) closes
apache/incubator-metron#96
METRON-122 Create generic unit test framework for testing grok statements (merrimanr) closes apache/incubator-metron#96
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/645d8292
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/645d8292
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/645d8292
Branch: refs/heads/master
Commit: 645d8292b5600d227b51bd80e73982a56d1cc77d
Parents: 28c250d
Author: merrimanr <me...@gmail.com>
Authored: Mon May 2 12:26:05 2016 -0500
Committer: rmerriman <rm...@hortonworks.com>
Committed: Mon May 2 12:26:05 2016 -0500
----------------------------------------------------------------------
.../common/configuration/EnrichmentConfig.java | 14 +-
.../configuration/SensorEnrichmentConfig.java | 4 +-
.../enrichment/bolt/EnrichmentSplitterBolt.java | 13 +-
.../bolt/ThreatIntelSplitterBolt.java | 14 +-
.../sample/data/SampleInput/BluecoatSyslog.txt | 144 +++++++++++++++++++
.../sample/data/SampleInput/SquidExampleOutput | 2 +
.../sample/data/SampleParsed/BluecoatParsed | 144 +++++++++++++++++++
.../sample/data/SampleParsed/SquidExampleParsed | 2 +
.../src/main/resources/sample/patterns/test | 2 +
metron-platform/metron-parsers/pom.xml | 11 ++
.../src/main/flux/bluecoat/remote.yaml | 71 +++++++++
.../src/main/flux/bluecoat/test.yaml | 72 ++++++++++
.../src/main/flux/squid/remote.yaml | 78 ++++++++++
.../src/main/flux/squid/test.yaml | 78 ++++++++++
.../metron-parsers/src/main/flux/yaf/test.yaml | 3 -
.../org/apache/metron/parsers/GrokParser.java | 37 +++--
.../parsers/bluecoat/BasicBluecoatParser.java | 101 +++++++++++++
.../src/main/resources/patterns/squid | 2 +
.../apache/metron/parsers/GrokParserTest.java | 63 +++-----
.../metron/parsers/SampleGrokParserTest.java | 82 +++++++++++
.../apache/metron/parsers/SquidParserTest.java | 75 ++++++++++
.../apache/metron/parsers/YafParserTest.java | 90 ++++++++++++
.../bluecoat/BasicBluecoatParserTest.java | 100 +++++++++++++
.../integration/BluecoatIntegrationTest.java | 48 +++++++
.../integration/ParserIntegrationTest.java | 14 +-
.../integration/SquidIntegrationTest.java | 49 +++++++
pom.xml | 1 +
27 files changed, 1246 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/645d8292/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfig.java
index 2ead81e..bcc91fa 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/EnrichmentConfig.java
@@ -22,6 +22,7 @@ import com.google.common.base.Joiner;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.common.Constants;
import org.apache.metron.common.cli.ConfigurationsUtils;
+import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -96,7 +97,14 @@ public class EnrichmentConfig {
}
@Override
public SensorEnrichmentConfig readConfig(String sensor) throws Exception {
- return SensorEnrichmentConfig.fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(sensor, client));
+ SensorEnrichmentConfig sensorEnrichmentConfig = new SensorEnrichmentConfig();
+ try {
+ sensorEnrichmentConfig = SensorEnrichmentConfig.fromBytes(ConfigurationsUtils.readSensorEnrichmentConfigBytesFromZookeeper(sensor, client));
+ }catch (KeeperException.NoNodeException e) {
+ sensorEnrichmentConfig.setIndex(sensor);
+ sensorEnrichmentConfig.setBatchSize(1);
+ }
+ return sensorEnrichmentConfig;
}
@Override
@@ -125,6 +133,8 @@ public class EnrichmentConfig {
fieldMap = config.getThreatIntelFieldMap();
if(fieldMap!= null) {
fieldList = fieldMap.get(Constants.SIMPLE_HBASE_THREAT_INTEL);
+ } else {
+ fieldMap = new HashMap<>();
}
if(fieldList == null) {
fieldList = new ArrayList<>();
@@ -140,6 +150,8 @@ public class EnrichmentConfig {
fieldMap = config.getEnrichmentFieldMap();
if(fieldMap!= null) {
fieldList = fieldMap.get(Constants.SIMPLE_HBASE_ENRICHMENT);
+ } else {
+ fieldMap = new HashMap<>();
}
if(fieldList == null) {
fieldList = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/645d8292/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorEnrichmentConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorEnrichmentConfig.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorEnrichmentConfig.java
index bc30327..6a45ec9 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorEnrichmentConfig.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/SensorEnrichmentConfig.java
@@ -28,8 +28,8 @@ import java.util.Map;
public class SensorEnrichmentConfig {
private String index;
- private Map<String, List<String>> enrichmentFieldMap;
- private Map<String, List<String>> threatIntelFieldMap;
+ private Map<String, List<String>> enrichmentFieldMap = new HashMap<>();
+ private Map<String, List<String>> threatIntelFieldMap = new HashMap<>();
private Map<String, List<String>> fieldToEnrichmentTypeMap = new HashMap<>();
private Map<String, List<String>> fieldToThreatIntelTypeMap = new HashMap<>();
private int batchSize;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/645d8292/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
index e713d69..6b49edb 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/EnrichmentSplitterBolt.java
@@ -21,6 +21,7 @@ import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Tuple;
import org.apache.metron.common.Constants;
+import org.apache.metron.common.configuration.SensorEnrichmentConfig;
import org.apache.metron.enrichment.configuration.Enrichment;
import org.apache.metron.enrichment.utils.EnrichmentUtils;
import org.apache.metron.common.utils.MessageUtils;
@@ -123,7 +124,17 @@ public class EnrichmentSplitterBolt extends SplitBolt<JSONObject> {
}
protected Map<String, List<String>> getFieldMap(String sensorType) {
- return configurations.getSensorEnrichmentConfig(sensorType).getEnrichmentFieldMap();
+ if(sensorType != null) {
+ SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sensorType);
+ if (config != null) {
+ return config.getEnrichmentFieldMap();
+ } else {
+ LOG.error("Unable to retrieve a sensor enrichment config of " + sensorType);
+ }
+ } else {
+ LOG.error("Trying to retrieve a field map with sensor type of null");
+ }
+ return new HashMap<>();
}
protected String getKeyName(String type, String field) {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/645d8292/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
index 692c327..1429b2c 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/bolt/ThreatIntelSplitterBolt.java
@@ -17,8 +17,10 @@
*/
package org.apache.metron.enrichment.bolt;
+import org.apache.metron.common.configuration.SensorEnrichmentConfig;
import org.apache.metron.enrichment.utils.ThreatIntelUtils;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -30,7 +32,17 @@ public class ThreatIntelSplitterBolt extends EnrichmentSplitterBolt {
@Override
protected Map<String, List<String>> getFieldMap(String sensorType) {
- return configurations.getSensorEnrichmentConfig(sensorType).getThreatIntelFieldMap();
+ if (sensorType != null) {
+ SensorEnrichmentConfig config = configurations.getSensorEnrichmentConfig(sensorType);
+ if (config != null) {
+ return config.getThreatIntelFieldMap();
+ } else {
+ LOG.error("Unable to retrieve sensor config: " + sensorType);
+ }
+ } else {
+ LOG.error("Trying to retrieve a field map with sensor type of null");
+ }
+ return new HashMap<>();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/645d8292/metron-platform/metron-integration-test/src/main/resources/sample/data/SampleInput/BluecoatSyslog.txt
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/resources/sample/data/SampleInput/BluecoatSyslog.txt b/metron-platform/metron-integration-test/src/main/resources/sample/data/SampleInput/BluecoatSyslog.txt
new file mode 100644
index 0000000..9738691
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/resources/sample/data/SampleInput/BluecoatSyslog.txt
@@ -0,0 +1,144 @@
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.29.36: user 'WJS310' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.216.222: user 'yaw983' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.216.222: user 'yaw983' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.216.222: user 'yaw983' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.114.217.29: user 'ags432' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.113.216.196: user 'u62206' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.127.221.164: user 'CXI886' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.127.221.164: user 'CXI886' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.114.217.29: user 'ags432' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.218.165.248: user 'fjl928' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'LOCAL\uzl193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.152.102.72: user 'ugs662' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.119.223.52: user 'XGZ521' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250018 LDAP: invalid credentials: reason: '80090308: LdapErr: DSID-0C0903AA, comment: AcceptSecurityContext error, data 52e, v1772' dn: 'CN=FJL928,OU=Developers,OU=All Users,DC=cof,DC=ds,DC=capitalone,DC=com' realm: 'AD_ldap'(2425130) NORMAL_EVENT realm_ldap.cpp 2833
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.218.165.248: user 'fjl928' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250018 LDAP: invalid credentials: reason: '80090308: LdapErr: DSID-0C0903AA, comment: AcceptSecurityContext error, data 52e, v1772' dn: 'CN=UZL193,OU=User Lock Policy 00,OU=All Users,DC=cof,DC=ds,DC=capitalone,DC=com' realm: 'AD_ldap'(2425130) NORMAL_EVENT realm_ldap.cpp 2833
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.119.223.52: user 'XGZ521' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.152.102.72: user 'ugs662' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250018 LDAP: invalid credentials: reason: '80090308: LdapErr: DSID-0C0903AA, comment: AcceptSecurityContext error, data 52e, v1772' dn: 'CN=EPL857,OU=User Lock Policy 05,OU=All Users,DC=cof,DC=ds,DC=capitalone,DC=com' realm: 'AD_ldap'(2425130) NORMAL_EVENT realm_ldap.cpp 2833
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.210.223.65: user 'epl857' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.210.223.65: user 'epl857' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.127.216.106: user 'LOCAL\sdq302' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'LOCAL\kon313' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250018 LDAP: invalid credentials: reason: '80090308: LdapErr: DSID-0C0903AA, comment: AcceptSecurityContext error, data 52e, v1772' dn: 'CN=SDQ302,OU=Developers,OU=All Users,DC=cof,DC=ds,DC=capitalone,DC=com' realm: 'AD_ldap'(2425130) NORMAL_EVENT realm_ldap.cpp 2833
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.127.216.106: user 'sdq302' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.120.144.20: user 'LOCAL\dkg773' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.120.144.20: user 'dkg773' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.120.144.20: user 'dkg773' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.152.102.72: user 'ugs662' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.219.193: user 'LOCAL\uua398' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.152.102.72: user 'ugs662' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:06 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:06 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.119.223.52: user 'XGZ521' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.29.36: user 'LOCAL\wjs310' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250018 LDAP: invalid credentials: reason: '80090308: LdapErr: DSID-0C0903AA, comment: AcceptSecurityContext error, data 52e, v1772' dn: 'CN=WJS310,OU=Developers,OU=All Users,DC=cof,DC=ds,DC=capitalone,DC=com' realm: 'AD_ldap'(2425130) NORMAL_EVENT realm_ldap.cpp 2833
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.29.36: user 'WJS310' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.29.36: user 'WJS310' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.216.222: user 'LOCAL\yaw983' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.216.222: user 'yaw983' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.216.222: user 'yaw983' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.114.217.29: user 'ags432' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.113.216.196: user 'LOCAL\u62206' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.115.220.223: user 'qwn225' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250001 LDAP: Authentication failed from 10.113.216.196: no such user in realm 'AD_ldap'(102089) NORMAL_EVENT realm_ldap.cpp 2634
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.114.217.29: user 'ags432' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.218.165.248: user 'LOCAL\fjl928' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.218.165.248: user 'fjl928' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.152.102.72: user 'ugs662' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.119.223.52: user 'XGZ521' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.119.223.52: user 'XGZ521' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.119.223.52: user 'XGZ521' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.218.165.248: user 'fjl928' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.218.165.248: user 'fjl928' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.210.223.65: user 'LOCAL\epl857' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.119.223.52: user 'XGZ521' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.152.102.72: user 'ugs662' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.210.223.65: user 'epl857' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.210.223.65: user 'epl857' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.210.223.65: user 'epl857' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.127.216.106: user 'sdq302' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.127.216.106: user 'sdq302' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250018 LDAP: invalid credentials: reason: '80090308: LdapErr: DSID-0C0903AA, comment: AcceptSecurityContext error, data 52e, v1772' dn: 'CN=UUA398,OU=Developers,OU=All Users,DC=cof,DC=ds,DC=capitalone,DC=com' realm: 'AD_ldap'(2425130) NORMAL_EVENT realm_ldap.cpp 2833
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.219.193: user 'uua398' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'LOCAL\uzl193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.29.228: user 'UZL193' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.212.21.253: user 'vwv149' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.118.219.193: user 'uua398' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.219.15.104: user 'kon313' (realm AD_ldap)(0) NORMAL_EVENT authutility.cpp 113
+<29>Apr 14 20:31:07 ProxySG: 250017 Authentication failed from 10.113.216.196: user 'u62206' (realm iwa_realm)(0) NORMAL_EVENT authutility.cpp 113
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/645d8292/metron-platform/metron-integration-test/src/main/resources/sample/data/SampleInput/SquidExampleOutput
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/resources/sample/data/SampleInput/SquidExampleOutput b/metron-platform/metron-integration-test/src/main/resources/sample/data/SampleInput/SquidExampleOutput
new file mode 100644
index 0000000..ae70fb9
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/resources/sample/data/SampleInput/SquidExampleOutput
@@ -0,0 +1,2 @@
+1461576382.642 161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html
+1461576442.228 159 127.0.0.1 TCP_MISS/200 137183 GET http://www.nba.com/ - DIRECT/66.210.41.9 text/html
\ No newline at end of file