You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/05/16 18:07:43 UTC
[02/15] incubator-metron git commit: METRON-142 Simplify Parser
configuration (merrimanr via cestella) closes apache/incubator-metron#120
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/sample/data/yaf/indexed/YafIndexed
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/yaf/indexed/YafIndexed b/metron-platform/metron-integration-test/src/main/sample/data/yaf/indexed/YafIndexed
new file mode 100644
index 0000000..1c38406
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/yaf/indexed/YafIndexed
@@ -0,0 +1,10 @@
+{"adapter.threatinteladapter.end.ts":"1457102731219","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa001","index.elasticsearchwriter.ts":"1457102731220","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731206","adapter.hostfromjsonlistadapter.begin.ts":"1457102731185","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":44,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731185","threatintelsplitterbolt.splitter.ts":"1457102731207","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,
"adapter.threatinteladapter.begin.ts":"1457102731210","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AS","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731220","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.ho
st.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":10000000,"index.elasticsearchwriter.ts":"1457102731221","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731208","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitt
er.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988502,"adapter.threatinteladapter.begin.ts":"1457102731219","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731221","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":37299,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latit
ude":"test latitude","timestamp":1453994988502,"risn":0,"end_time":1453994988502,"is_alert":"true","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":37299,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":312,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter
.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988504,"enrichments.host.dip.known_i
nfo.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":
"1457102731211","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":56303,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"t
est latitude","timestamp":1453994988504,"risn":0,"end_time":1453994988504,"is_alert":"true","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":56303,"rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":84,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988506,"adapter.
threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988506,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988506,"enrichments.host.dip.known_in
fo.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fca","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":60,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbol
t.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988508,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"S","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":145399
4988508,"risn":0,"end_time":1453994988508,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterb
olt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453
994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":148,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitter
bolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":14
53994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731225","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":145399498851
2,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.h
ost.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731226","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":604,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731213","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988562
,"adapter.threatinteladapter.begin.ts":"1457102731226","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731226","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988562,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988562,"enrichments.h
ost.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/sample/data/yaf/parsed/YafExampleParsed
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/yaf/parsed/YafExampleParsed b/metron-platform/metron-integration-test/src/main/sample/data/yaf/parsed/YafExampleParsed
new file mode 100644
index 0000000..6155e98
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/yaf/parsed/YafExampleParsed
@@ -0,0 +1,10 @@
+{"iflags":"AS","uflags":0,"isn":"22efa001","ip_dst_addr":"10.0.2.15","ip_dst_port":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"216.21.170.221","tag":0,"rtag":0,"ip_src_port":80,"timestamp":1453994988512,"app":0,"oct":44,"end_reason":"idle","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","protocol":6}
+{"iflags":"A","uflags":0,"isn":10000000,"ip_dst_addr":"10.0.2.3","ip_dst_port":53,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":37299,"timestamp":1453994988502,"app":0,"oct":56,"end_reason":"idle","risn":0,"end_time":1453994988502,"source.type":"yaf","start_time":1453994988502,"riflags":0,"rtt":"0.000","protocol":17}
+{"iflags":"A","uflags":0,"isn":0,"ip_dst_addr":"10.0.2.15","ip_dst_port":37299,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.3","tag":0,"rtag":0,"ip_src_port":53,"timestamp":1453994988504,"app":0,"oct":312,"end_reason":"idle","risn":0,"end_time":1453994988504,"source.type":"yaf","start_time":1453994988504,"riflags":0,"rtt":"0.000","protocol":17}
+{"iflags":"A","uflags":0,"isn":0,"ip_dst_addr":"10.0.2.3","ip_dst_port":53,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":56303,"timestamp":1453994988504,"app":0,"oct":56,"end_reason":"idle","risn":0,"end_time":1453994988504,"source.type":"yaf","start_time":1453994988504,"riflags":0,"rtt":"0.000","protocol":17}
+{"iflags":"A","uflags":0,"isn":0,"ip_dst_addr":"10.0.2.15","ip_dst_port":56303,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.3","tag":0,"rtag":0,"ip_src_port":53,"timestamp":1453994988506,"app":0,"oct":84,"end_reason":"idle","risn":0,"end_time":1453994988506,"source.type":"yaf","start_time":1453994988506,"riflags":0,"rtt":"0.000","protocol":17}
+{"iflags":"S","uflags":0,"isn":"58c52fca","ip_dst_addr":"216.21.170.221","ip_dst_port":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":39468,"timestamp":1453994988508,"app":0,"oct":60,"end_reason":"idle","risn":0,"end_time":1453994988508,"source.type":"yaf","start_time":1453994988508,"riflags":0,"rtt":"0.000","protocol":6}
+{"iflags":"A","uflags":0,"isn":"58c52fcb","ip_dst_addr":"216.21.170.221","ip_dst_port":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":39468,"timestamp":1453994988512,"app":0,"oct":40,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","protocol":6}
+{"iflags":"AP","uflags":0,"isn":"58c52fcb","ip_dst_addr":"216.21.170.221","ip_dst_port":80,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"10.0.2.15","tag":0,"rtag":0,"ip_src_port":39468,"timestamp":1453994988512,"app":0,"oct":148,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","protocol":6}
+{"iflags":"A","uflags":0,"isn":"22efa002","ip_dst_addr":"10.0.2.15","ip_dst_port":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"216.21.170.221","tag":0,"rtag":0,"ip_src_port":80,"timestamp":1453994988512,"app":0,"oct":40,"end_reason":"idle ","risn":0,"end_time":1453994988512,"source.type":"yaf","start_time":1453994988512,"riflags":0,"rtt":"0.000","protocol":6}
+{"iflags":"AP","uflags":0,"isn":"22efa002","ip_dst_addr":"10.0.2.15","ip_dst_port":39468,"duration":"0.000","rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","pkt":1,"ruflags":0,"roct":0,"ip_src_addr":"216.21.170.221","tag":0,"rtag":0,"ip_src_port":80,"timestamp":1453994988562,"app":0,"oct":604,"end_reason":"idle","risn":0,"end_time":1453994988562,"source.type":"yaf","start_time":1453994988562,"riflags":0,"rtt":"0.000","protocol":6}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/sample/data/yaf/raw/YafExampleOutput
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/data/yaf/raw/YafExampleOutput b/metron-platform/metron-integration-test/src/main/sample/data/yaf/raw/YafExampleOutput
new file mode 100644
index 0000000..8f3ff44
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/data/yaf/raw/YafExampleOutput
@@ -0,0 +1,10 @@
+2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle
+2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle
+2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle
+2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle
+2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle
+2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle
+2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle
+2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle
+2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle
+2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-integration-test/src/main/sample/patterns/test
----------------------------------------------------------------------
diff --git a/metron-platform/metron-integration-test/src/main/sample/patterns/test b/metron-platform/metron-integration-test/src/main/sample/patterns/test
new file mode 100644
index 0000000..a88a255
--- /dev/null
+++ b/metron-platform/metron-integration-test/src/main/sample/patterns/test
@@ -0,0 +1,2 @@
+YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}
+YAF_DELIMITED %{NUMBER:start_time}\|%{YAF_TIME_FORMAT:end_time}\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\|%{SPACE:UNWANTED}%{INT:protocol}\|%{SPACE:UNWANTED}%{IP:ip_src_addr}\|%{SPACE:UNWANTED}%{INT:ip_src_port}\|%{SPACE:UNWANTED}%{IP:ip_dst_addr}\|%{SPACE:UNWANTED}%{INT:ip_dst_port}\|%{SPACE:UNWANTED}%{DATA:iflags}\|%{SPACE:UNWANTED}%{DATA:uflags}\|%{SPACE:UNWANTED}%{DATA:riflags}\|%{SPACE:UNWANTED}%{DATA:ruflags}\|%{SPACE:UNWANTED}%{WORD:isn}\|%{SPACE:UNWANTED}%{DATA:risn}\|%{SPACE:UNWANTED}%{DATA:tag}\|%{GREEDYDATA:rtag}\|%{SPACE:UNWANTED}%{INT:pkt}\|%{SPACE:UNWANTED}%{INT:oct}\|%{SPACE:UNWANTED}%{INT:rpkt}\|%{SPACE:UNWANTED}%{INT:roct}\|%{SPACE:UNWANTED}%{INT:app}\|%{GREEDYDATA:end_reason}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/config/parsers.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/parsers.properties b/metron-platform/metron-parsers/src/main/config/parsers.properties
deleted file mode 100644
index 7b906d2..0000000
--- a/metron-platform/metron-parsers/src/main/config/parsers.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-##### Kafka #####
-
-kafka.zk=node1:2181
-kafka.broker=node1:6667
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bluecoat.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bluecoat.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bluecoat.json
new file mode 100644
index 0000000..303bedc
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bluecoat.json
@@ -0,0 +1,5 @@
+{
+"parserClassName":"org.apache.metron.parsers.bluecoat.BasicBluecoatParser",
+"sensorTopic":"bluecoat",
+"parserConfig": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bro.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bro.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bro.json
new file mode 100644
index 0000000..a9750c2
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/bro.json
@@ -0,0 +1,5 @@
+{
+ "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser",
+ "sensorTopic":"bro",
+ "parserConfig": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/snort.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/snort.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/snort.json
new file mode 100644
index 0000000..be36fa2
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/snort.json
@@ -0,0 +1,5 @@
+{
+ "parserClassName":"org.apache.metron.parsers.snort.BasicSnortParser",
+ "sensorTopic":"snort",
+ "parserConfig": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json
new file mode 100644
index 0000000..6c4a69b
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/squid.json
@@ -0,0 +1,9 @@
+{
+ "parserClassName": "org.apache.metron.parsers.GrokParser",
+ "sensorTopic": "squid",
+ "parserConfig": {
+ "grokPath": "/patterns/squid",
+ "patternLabel": "SQUID_DELIMITED",
+ "timestampField": "timestamp"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json
new file mode 100644
index 0000000..0f2c901
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/websphere.json
@@ -0,0 +1,11 @@
+{
+ "parserClassName":"org.apache.metron.parsers.websphere.GrokWebSphereParser",
+ "sensorTopic":"websphere",
+ "parserConfig":
+ {
+ "grokPath":"/patterns/websphere",
+ "patternLabel":"WEBSPHERE",
+ "timestampField":"timestamp_string",
+ "dateFormat":"yyyy MMM dd HH:mm:ss"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/yaf.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/yaf.json b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/yaf.json
new file mode 100644
index 0000000..6290e9f
--- /dev/null
+++ b/metron-platform/metron-parsers/src/main/config/zookeeper/parsers/yaf.json
@@ -0,0 +1,12 @@
+{
+ "parserClassName":"org.apache.metron.parsers.GrokParser",
+ "sensorTopic":"yaf",
+ "parserConfig":
+ {
+ "grokPath":"/patterns/yaf",
+ "patternLabel":"YAF_DELIMITED",
+ "timestampField":"start_time",
+ "timeFields": ["start_time", "end_time"],
+ "dateFormat":"yyyy-MM-dd HH:mm:ss.S"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/bluecoat/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/bluecoat/remote.yaml b/metron-platform/metron-parsers/src/main/flux/bluecoat/remote.yaml
deleted file mode 100644
index 1f2cd14..0000000
--- a/metron-platform/metron-parsers/src/main/flux/bluecoat/remote.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "bluecoat"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.bluecoat.BasicBluecoatParser"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "bluecoat"
- # zk root
- - ""
- # id
- - "bluecoat"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "startOffsetTime"
- value: -1
- - name: "socketTimeoutMs"
- value: 1000000
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "bluecoat"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/bluecoat/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/bluecoat/test.yaml b/metron-platform/metron-parsers/src/main/flux/bluecoat/test.yaml
deleted file mode 100644
index f1016e6..0000000
--- a/metron-platform/metron-parsers/src/main/flux/bluecoat/test.yaml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "bluecoat-test"
-config:
- topology.workers: 1
-
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.bluecoat.BasicBluecoatParser"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "bluecoat"
- # zk root
- - ""
- # id
- - "bluecoat"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "startOffsetTime"
- value: -2
- - name: "socketTimeoutMs"
- value: 1000000
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "bluecoat"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/bro/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/bro/remote.yaml b/metron-platform/metron-parsers/src/main/flux/bro/remote.yaml
deleted file mode 100644
index 1852499..0000000
--- a/metron-platform/metron-parsers/src/main/flux/bro/remote.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "bro"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.bro.BasicBroParser"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "bro"
- # zk root
- - ""
- # id
- - "bro"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "startOffsetTime"
- value: -1
- - name: "socketTimeoutMs"
- value: 1000000
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "bro"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/bro/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/bro/test.yaml b/metron-platform/metron-parsers/src/main/flux/bro/test.yaml
deleted file mode 100644
index 42c3261..0000000
--- a/metron-platform/metron-parsers/src/main/flux/bro/test.yaml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "bro-test"
-config:
- topology.workers: 1
-
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.bro.BasicBroParser"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "bro"
- # zk root
- - ""
- # id
- - "bro"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "startOffsetTime"
- value: -2
- - name: "socketTimeoutMs"
- value: 1000000
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "bro"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/snort/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/snort/remote.yaml b/metron-platform/metron-parsers/src/main/flux/snort/remote.yaml
deleted file mode 100644
index 8317acf..0000000
--- a/metron-platform/metron-parsers/src/main/flux/snort/remote.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "snort"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.snort.BasicSnortParser"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "snort"
- # zk root
- - ""
- # id
- - "snort"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "startOffsetTime"
- value: -1
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "snort"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/snort/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/snort/test.yaml b/metron-platform/metron-parsers/src/main/flux/snort/test.yaml
deleted file mode 100644
index 5b9a2df..0000000
--- a/metron-platform/metron-parsers/src/main/flux/snort/test.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "snort-test"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.snort.BasicSnortParser"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "snort"
- # zk root
- - ""
- # id
- - "snort"
- properties:
- - name: "ignoreZkOffsets"
- value: false
- - name: "startOffsetTime"
- value: -2
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "snort"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/squid/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/squid/remote.yaml b/metron-platform/metron-parsers/src/main/flux/squid/remote.yaml
deleted file mode 100644
index 119f03e..0000000
--- a/metron-platform/metron-parsers/src/main/flux/squid/remote.yaml
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "squid"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.GrokParser"
- constructorArgs:
- - "/apps/metron/patterns/squid"
- - "SQUID_DELIMITED"
- configMethods:
- - name: "withTimestampField"
- args:
- - "timestamp"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "squid"
- # zk root
- - ""
- # id
- - "squid"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "startOffsetTime"
- value: -1
- - name: "socketTimeoutMs"
- value: 1000000
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "squid"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/squid/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/squid/test.yaml b/metron-platform/metron-parsers/src/main/flux/squid/test.yaml
deleted file mode 100644
index 77893d2..0000000
--- a/metron-platform/metron-parsers/src/main/flux/squid/test.yaml
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "squid"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.GrokParser"
- constructorArgs:
- - "../metron-parsers/src/main/resources/patterns/squid"
- - "SQUID_DELIMITED"
- configMethods:
- - name: "withTimestampField"
- args:
- - "timestamp"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "squid"
- # zk root
- - ""
- # id
- - "squid"
- properties:
- - name: "ignoreZkOffsets"
- value: false
- - name: "startOffsetTime"
- value: -2
- - name: "socketTimeoutMs"
- value: 1000000
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "squid"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/yaf/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/yaf/remote.yaml b/metron-platform/metron-parsers/src/main/flux/yaf/remote.yaml
deleted file mode 100644
index f50b319..0000000
--- a/metron-platform/metron-parsers/src/main/flux/yaf/remote.yaml
+++ /dev/null
@@ -1,84 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "yaf"
-config:
- topology.workers: 1
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.GrokParser"
- constructorArgs:
- - "/patterns/yaf"
- - "YAF_DELIMITED"
- configMethods:
- - name: "withTimestampField"
- args:
- - "start_time"
- - name: "withTimeFields"
- args:
- - ["start_time", "end_time"]
- - name: "withDateFormat"
- args:
- - "yyyy-MM-dd HH:mm:ss.S"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "yaf"
- # zk root
- - ""
- # id
- - "yaf"
- properties:
- - name: "ignoreZkOffsets"
- value: true
- - name: "startOffsetTime"
- value: -1
- - name: "socketTimeoutMs"
- value: 1000000
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "yaf"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/flux/yaf/test.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/flux/yaf/test.yaml b/metron-platform/metron-parsers/src/main/flux/yaf/test.yaml
deleted file mode 100644
index e2985b8..0000000
--- a/metron-platform/metron-parsers/src/main/flux/yaf/test.yaml
+++ /dev/null
@@ -1,85 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "yaf-test"
-config:
- topology.workers: 1
-
-
-components:
- - id: "parser"
- className: "org.apache.metron.parsers.GrokParser"
- constructorArgs:
- - "../metron-parsers/src/main/resources/patterns/yaf"
- - "YAF_DELIMITED"
- configMethods:
- - name: "withTimestampField"
- args:
- - "start_time"
- - name: "withTimeFields"
- args:
- - ["start_time", "end_time"]
- - name: "withDateFormat"
- args:
- - "yyyy-MM-dd HH:mm:ss.S"
- - id: "writer"
- className: "org.apache.metron.parsers.writer.KafkaWriter"
- constructorArgs:
- - "${kafka.broker}"
- - id: "zkHosts"
- className: "storm.kafka.ZkHosts"
- constructorArgs:
- - "${kafka.zk}"
- - id: "kafkaConfig"
- className: "storm.kafka.SpoutConfig"
- constructorArgs:
- # zookeeper hosts
- - ref: "zkHosts"
- # topic name
- - "yaf"
- # zk root
- - ""
- # id
- - "yaf"
- properties:
- - name: "ignoreZkOffsets"
- value: false
- - name: "startOffsetTime"
- value: -2
- - name: "socketTimeoutMs"
- value: 1000000
-
-spouts:
- - id: "kafkaSpout"
- className: "storm.kafka.KafkaSpout"
- constructorArgs:
- - ref: "kafkaConfig"
-
-bolts:
- - id: "parserBolt"
- className: "org.apache.metron.parsers.bolt.ParserBolt"
- constructorArgs:
- - "${kafka.zk}"
- - "yaf"
- - ref: "parser"
- - ref: "writer"
-
-streams:
- - name: "spout -> bolt"
- from: "kafkaSpout"
- to: "parserBolt"
- grouping:
- type: SHUFFLE
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
index 2b92ffb..0379080 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/GrokParser.java
@@ -39,6 +39,7 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import java.util.TimeZone;
public class GrokParser implements MessageParser<JSONObject>, Serializable {
@@ -46,61 +47,42 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
protected static final Logger LOG = LoggerFactory.getLogger(GrokParser.class);
protected transient Grok grok;
- protected String grokHdfsPath;
+ protected String grokPath;
protected String patternLabel;
- protected String[] timeFields = new String[0];
+ protected List<String> timeFields = new ArrayList<>();
protected String timestampField;
protected SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z");
- protected TimeZone timeZone = TimeZone.getTimeZone("UTC");
protected String patternsCommonDir = "/patterns/common";
- public GrokParser(String grokHdfsPath, String patternLabel) {
- this.grokHdfsPath = grokHdfsPath;
- this.patternLabel = patternLabel;
- }
-
- public GrokParser withTimestampField(String timestampField) {
- this.timestampField = timestampField;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Grok parser settting timestamp field: " + timestampField);
- }
- return this;
- }
-
- public GrokParser withTimeFields(String... timeFields) {
- this.timeFields = timeFields;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Grok parser settting time fields: " + timeFields);
- }
- return this;
- }
-
- public GrokParser withDateFormat(String dateFormat) {
- this.dateFormat = new SimpleDateFormat(dateFormat);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Grok parser settting date format: " + dateFormat);
- }
- return this;
- }
-
- public GrokParser withTimeZone(String timeZone) {
- this.timeZone = TimeZone.getTimeZone(timeZone);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Grok parser settting timezone: " + timeZone);
+ @Override
+ public void configure(Map<String, Object> parserConfig) {
+ this.grokPath = (String) parserConfig.get("grokPath");
+ this.patternLabel = (String) parserConfig.get("patternLabel");
+ this.timestampField = (String) parserConfig.get("timestampField");
+ List<String> timeFieldsParam = (List<String>) parserConfig.get("timeFields");
+ if (timeFieldsParam != null) {
+ this.timeFields = timeFieldsParam;
+ }
+ String dateFormatParam = (String) parserConfig.get("dateFormat");
+ if (dateFormatParam != null) {
+ this.dateFormat = new SimpleDateFormat(dateFormatParam);
+ }
+ String timeZoneParam = (String) parserConfig.get("timeZone");
+ if (timeZoneParam != null) {
+ dateFormat.setTimeZone(TimeZone.getTimeZone(timeZoneParam));
+ } else {
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
}
- return this;
}
public InputStream openInputStream(String streamName) throws IOException {
- InputStream is = getClass().getResourceAsStream(streamName);
- if(is == null) {
- FileSystem fs = FileSystem.get(new Configuration());
- Path path = new Path(streamName);
- if(fs.exists(path)) {
- return fs.open(path);
- }
+ FileSystem fs = FileSystem.get(new Configuration());
+ Path path = new Path(streamName);
+ if(fs.exists(path)) {
+ return fs.open(path);
+ } else {
+ return getClass().getResourceAsStream(streamName);
}
- return is;
}
@Override
@@ -119,12 +101,12 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
grok.addPatternFromReader(new InputStreamReader(commonInputStream));
if (LOG.isDebugEnabled()) {
- LOG.debug("Loading parser-specific patterns from: " + grokHdfsPath);
+ LOG.debug("Loading parser-specific patterns from: " + grokPath);
}
- InputStream patterInputStream = openInputStream(grokHdfsPath);
+ InputStream patterInputStream = openInputStream(grokPath);
if (patterInputStream == null) {
- throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + grokHdfsPath
+ throw new RuntimeException("Grok parser unable to initialize grok parser: Unable to load " + grokPath
+ " from either classpath or HDFS");
}
grok.addPatternFromReader(new InputStreamReader(patterInputStream));
@@ -166,7 +148,7 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
if (message.size() == 0)
throw new RuntimeException("Grok statement produced a null message. Original message was: "
+ originalMessage + " and the parsed message was: " + message + " . Check the pattern at: "
- + grokHdfsPath);
+ + grokPath);
message.put("original_string", originalMessage);
for (String timeField : timeFields) {
@@ -222,7 +204,6 @@ public class GrokParser implements MessageParser<JSONObject>, Serializable {
LOG.debug("Grok perser converting timestamp to epoch: " + datetime);
}
- dateFormat.setTimeZone(timeZone);
Date date = dateFormat.parse(datetime);
if (LOG.isDebugEnabled()) {
LOG.debug("Grok perser converted timestamp to epoch: " + date);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java
index 0f8a862..4f1c8b0 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/asa/GrokAsaParser.java
@@ -186,7 +186,12 @@ public class GrokAsaParser extends BasicParser {
return timeInMillisSinceEpoch;
}
-
+
+ @Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
@Override
public void init() {
// pattern_url = Resources.getResource("patterns/asa");
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bluecoat/BasicBluecoatParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bluecoat/BasicBluecoatParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bluecoat/BasicBluecoatParser.java
index 9d9ab6b..fcefcc2 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bluecoat/BasicBluecoatParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bluecoat/BasicBluecoatParser.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
+import java.util.Map;
import org.apache.metron.parsers.BasicParser;
import org.json.simple.JSONObject;
@@ -34,7 +35,12 @@ public class BasicBluecoatParser extends BasicParser {
private static final Logger _LOG = LoggerFactory.getLogger(BasicBluecoatParser.class);
private SimpleDateFormat df = new SimpleDateFormat("MMM dd yyyy HH:mm:ss");
-
+
+ @Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
@Override
public void init() {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index f0f1bd8..39f2641 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -23,6 +23,7 @@ import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Tuple;
import org.apache.metron.common.Constants;
import org.apache.metron.common.bolt.ConfiguredBolt;
+import org.apache.metron.common.bolt.ConfiguredParserBolt;
import org.apache.metron.parsers.filters.GenericMessageFilter;
import org.apache.metron.common.utils.ErrorUtils;
import org.apache.metron.parsers.interfaces.MessageFilter;
@@ -33,7 +34,7 @@ import org.json.simple.JSONObject;
import java.util.List;
import java.util.Map;
-public class ParserBolt extends ConfiguredBolt {
+public class ParserBolt extends ConfiguredParserBolt {
private OutputCollector collector;
private MessageParser<JSONObject> parser;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
index 74ea082..4052e86 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bro/BasicBroParser.java
@@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
@SuppressWarnings("serial")
public class BasicBroParser extends BasicParser {
@@ -36,6 +37,11 @@ public class BasicBroParser extends BasicParser {
private JSONCleaner cleaner = new JSONCleaner();
@Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
+ @Override
public void init() {
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
index b90d2b7..498248a 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/fireeye/BasicFireEyeParser.java
@@ -31,6 +31,7 @@ import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -59,6 +60,11 @@ public class BasicFireEyeParser extends BasicParser {
}
@Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
+ @Override
public void init() {
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
index 11efa53..81d1b1a 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/interfaces/MessageParser.java
@@ -17,10 +17,13 @@
*/
package org.apache.metron.parsers.interfaces;
+import java.io.Serializable;
import java.util.List;
+import java.util.Map;
-public interface MessageParser<T> {
+public interface MessageParser<T> extends Serializable {
+ void configure(Map<String, Object> parserConfig);
void init();
List<T> parse(byte[] rawMessage);
boolean validate(T message);
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
index 19b3ac6..2d559ac 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/ise/BasicIseParser.java
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
@SuppressWarnings("serial")
public class BasicIseParser extends BasicParser {
@@ -37,6 +38,11 @@ public class BasicIseParser extends BasicParser {
static final transient ISEParser _parser = new ISEParser("header=");
@Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
+ @Override
public void init() {
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
index 6c25d67..83eedcc 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/lancope/BasicLancopeParser.java
@@ -28,6 +28,7 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Map;
@SuppressWarnings("serial")
public class BasicLancopeParser extends BasicParser {
@@ -38,6 +39,11 @@ public class BasicLancopeParser extends BasicParser {
.class);
@Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
+ @Override
public void init() {
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
index 39177aa..2f5310c 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/logstash/BasicLogstashParser.java
@@ -24,10 +24,16 @@ import org.json.simple.parser.JSONParser;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
public class BasicLogstashParser extends BasicParser {
@Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
+ @Override
public void init() {
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
index c67e2b5..e6b9274 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
@@ -27,6 +27,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
public class BasicPaloAltoFirewallParser extends BasicParser {
@@ -90,6 +91,10 @@ public class BasicPaloAltoFirewallParser extends BasicParser {
public static final String PktsSent = "pkts_sent";
public static final String PktsReceived = "pkts_received";
+ @Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
@Override
public void init() {
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
index a82d8d6..1fcb6c4 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/snort/BasicSnortParser.java
@@ -73,6 +73,11 @@ public class BasicSnortParser extends BasicParser {
private String recordDelimiter = ",";
@Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
+ @Override
public void init() {
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/df8d682e/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
index 40badcd..0bc2671 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/sourcefire/BasicSourcefireParser.java
@@ -25,6 +25,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -42,6 +43,11 @@ public class BasicSourcefireParser extends BasicParser {
Pattern pattern = Pattern.compile(domain_name_regex);
@Override
+ public void configure(Map<String, Object> parserConfig) {
+
+ }
+
+ @Override
public void init() {
}