You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/03/21 18:06:05 UTC
[23/43] incubator-metron git commit: METRON-58 Remediate Deployment
Integration Testing Issues (dlyle65535 via cestella) closes
apache/incubator-metron#36
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed b/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
index 27b3589..1c38406 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleIndexed/YafIndexed
@@ -1,10 +1,10 @@
-{enrichments.geo.dip.longitude=test longitude, iflags=AS, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa001, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
=80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, oct=44, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988512, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988512, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}
-{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=10000000, dip=10.0.2.3, enrichments.host.sip.known_info.local=YES, dp=53, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longit
ude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=37299, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988502, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=56, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988502, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988502, riflags=0, rtt=0.000, threatintels.ip.dip.threat_source=ip_threat_intel, proto=17}
-{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=0, dip=10.0.2.15, dp=37299, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.3, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=53, enrichmen
ts.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988504, app=0, oct=312, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988504, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988504, threatintels.ip.sip.threat_source=ip_threat_intel, riflags=0, rtt=0.000, proto=17, enrichments.host.dip.known_info.local=YES}
-{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=0, dip=10.0.2.3, enrichments.host.sip.known_info.local=YES, dp=53, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,tes
t latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=56303, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988504, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=56, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988504, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988504, riflags=0, rtt=0.000, threatintels.ip.dip.threat_source=ip_threat_intel, proto=17}
-{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=0, dip=10.0.2.15, dp=56303, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.3, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=53, enrichmen
ts.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988506, app=0, oct=84, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988506, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988506, threatintels.ip.sip.threat_source=ip_threat_intel, riflags=0, rtt=0.000, proto=17, enrichments.host.dip.known_info.local=YES}
-{enrichments.geo.dip.longitude=test longitude, iflags=S, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fca, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.si
p.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988508, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=60, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, end_time=1453994988508, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988508, riflags=0, rtt=0.000, proto=6}
-{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fcb, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.s
ip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=40, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, end_time=1453994988512, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988512, riflags=0, rtt=0.000, proto=6}
-{enrichments.geo.dip.longitude=test longitude, iflags=AP, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, enrichments.host.sip.known_info.asset_value=important, isn=58c52fcb, dip=216.21.170.221, enrichments.host.sip.known_info.local=YES, dp=80, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.host.sip.known_info.type=printer, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.
sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=10.0.2.15, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp=39468, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, enrichments.host.dip=, oct=148, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, end_time=1453994988512, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, start_time=1453994988512, riflags=0, rtt=0.000, proto=6}
-{enrichments.geo.dip.longitude=test longitude, iflags=A, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa002, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle , enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
=80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988512, app=0, threatintels.ip.sip=, oct=40, end_reason=idle , enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988512, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988512, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}
-{enrichments.geo.dip.longitude=test longitude, iflags=AP, enrichments.geo.dip.location_point=test longitude,test latitude, uflags=0, isn=22efa002, dip=10.0.2.15, dp=39468, threatintels.ip.dip=, enrichments.geo.sip.postalCode=test postalCode, duration=0.000, rpkt=0, enrichments.geo.dip.country=test country, original_string=2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle, enrichments.geo.dip.locID=1, enrichments.geo.sip.city=test city, enrichments.geo.dip.latitude=test latitude, enrichments.geo.sip.country=test country, enrichments.geo.dip.city=test city, enrichments.geo.sip.dmaCode=test dmaCode, pkt=1, enrichments.geo.sip.location_point=test longitude,test latitude, ruflags=0, roct=0, sip=216.21.170.221, tag=0, enrichments.geo.dip.dmaCode=test dmaCode, rtag=0, sp
=80, enrichments.geo.sip.longitude=test longitude, enrichments.geo.sip.latitude=test latitude, timestamp=1453994988562, app=0, threatintels.ip.sip=, oct=604, end_reason=idle, enrichments.geo.sip.locID=1, risn=0, enrichments.host.dip.known_info.type=printer, end_time=1453994988562, enrichments.host.dip.known_info.asset_value=important, enrichments.geo.dip.postalCode=test postalCode, source.type=yaf, enrichments.host.sip=, start_time=1453994988562, riflags=0, rtt=0.000, proto=6, enrichments.host.dip.known_info.local=YES}
+{"adapter.threatinteladapter.end.ts":"1457102731219","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa001","index.elasticsearchwriter.ts":"1457102731220","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AS| 0| 0| 0|22efa001|00000000|000|000| 1| 44| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731206","adapter.hostfromjsonlistadapter.begin.ts":"1457102731185","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":44,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731185","threatintelsplitterbolt.splitter.ts":"1457102731207","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,
"adapter.threatinteladapter.begin.ts":"1457102731210","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AS","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731220","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.ho
st.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":10000000,"index.elasticsearchwriter.ts":"1457102731221","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.502|2016-01-28 15:29:48.502| 0.000| 0.000| 17| 10.0.2.15|37299| 10.0.2.3| 53| A| 0| 0| 0|10000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731208","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitt
er.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988502,"adapter.threatinteladapter.begin.ts":"1457102731219","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731198","adapter.hostfromjsonlistadapter.end.ts":"1457102731197","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731221","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":37299,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latit
ude":"test latitude","timestamp":1453994988502,"risn":0,"end_time":1453994988502,"is_alert":"true","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731221","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":37299,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|37299| A| 0| 0| 0|00000000|00000000|000|000| 1| 312| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731197","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":312,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731198","threatintelsplitterbolt.splitter.ts":"1457102731210","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter
.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988504,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988504,"enrichments.host.dip.known_i
nfo.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.3","dp":53,"rpkt":0,"original_string":"2016-01-28 15:29:48.504|2016-01-28 15:29:48.504| 0.000| 0.000| 17| 10.0.2.15|56303| 10.0.2.3| 53| A| 0| 0| 0|00000000|00000000|000|000| 1| 56| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731209","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":56,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":
"1457102731211","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988504,"adapter.threatinteladapter.begin.ts":"1457102731221","riflags":0,"proto":17,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","threatintels.ip.dip.ip_threat_intel":"alert","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":56303,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"t
est latitude","timestamp":1453994988504,"risn":0,"end_time":1453994988504,"is_alert":"true","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":0,"index.elasticsearchwriter.ts":"1457102731222","dip":"10.0.2.15","dp":56303,"rpkt":0,"original_string":"2016-01-28 15:29:48.506|2016-01-28 15:29:48.506| 0.000| 0.000| 17| 10.0.2.3| 53| 10.0.2.15|56303| A| 0| 0| 0|00000000|00000000|000|000| 1| 84| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":84,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988506,"adapter.
threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":17,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731222","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.3","rtag":0,"sp":53,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988506,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988506,"enrichments.host.dip.known_in
fo.asset_value":"important","is_alert":"true","source.type":"yaf","threatintels.ip.sip.ip_threat_intel":"alert","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731222","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fca","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.508|2016-01-28 15:29:48.508| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| S| 0| 0| 0|58c52fca|00000000|000|000| 1| 60| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":60,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbol
t.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988508,"adapter.threatinteladapter.begin.ts":"1457102731222","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"S","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":145399
4988508,"risn":0,"end_time":1453994988508,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| A| 0| 0| 0|58c52fcb|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterb
olt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731223","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453
994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731223","enrichments.geo.dip.location_point":"test longitude,test latitude","enrichments.host.sip.known_info.asset_value":"important","isn":"58c52fcb","index.elasticsearchwriter.ts":"1457102732038","dip":"216.21.170.221","dp":80,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 10.0.2.15|39468| 216.21.170.221| 80| AP| 0| 0| 0|58c52fcb|00000000|000|000| 1| 148| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichments.host.sip.known_info.type":"printer","enrichmentjoinbolt.joiner.ts":"1457102731210","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":148,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitter
bolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988512,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.host.sip.known_info.local":"YES","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"10.0.2.15","rtag":0,"sp":39468,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":14
53994988512,"risn":0,"end_time":1453994988512,"source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731225","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.512|2016-01-28 15:29:48.512| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| A| 0| 0| 0|22efa002|00000000|000|000| 1| 40| 0| 0| 0|idle ","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":40,"end_reason":"idle ","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731212","enrichments.geo.dip.postalCode":"test postalCode","start_time":145399498851
2,"adapter.threatinteladapter.begin.ts":"1457102731223","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"A","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731225","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988512,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988512,"enrichments.h
ost.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
+{"adapter.threatinteladapter.end.ts":"1457102731226","enrichments.geo.dip.location_point":"test longitude,test latitude","isn":"22efa002","index.elasticsearchwriter.ts":"1457102732038","dip":"10.0.2.15","dp":39468,"rpkt":0,"original_string":"2016-01-28 15:29:48.562|2016-01-28 15:29:48.562| 0.000| 0.000| 6| 216.21.170.221| 80| 10.0.2.15|39468| AP| 0| 0| 0|22efa002|00000000|000|000| 1| 604| 0| 0| 0|idle","enrichments.geo.dip.locID":"1","enrichments.geo.sip.city":"test city","enrichmentjoinbolt.joiner.ts":"1457102731211","adapter.hostfromjsonlistadapter.begin.ts":"1457102731198","tag":0,"enrichments.geo.dip.dmaCode":"test dmaCode","app":0,"oct":604,"end_reason":"idle","enrichments.geo.sip.locID":"1","adapter.mockgeoadapter.begin.ts":"1457102731199","threatintelsplitterbolt.splitter.ts":"1457102731213","enrichments.geo.dip.postalCode":"test postalCode","start_time":1453994988562
,"adapter.threatinteladapter.begin.ts":"1457102731226","riflags":0,"proto":6,"enrichments.host.dip.known_info.local":"YES","enrichments.geo.dip.longitude":"test longitude","iflags":"AP","uflags":0,"adapter.mockgeoadapter.end.ts":"1457102731199","adapter.hostfromjsonlistadapter.end.ts":"1457102731198","enrichments.geo.sip.postalCode":"test postalCode","duration":"0.000","enrichments.geo.dip.country":"test country","threatinteljoinbolt.joiner.ts":"1457102731226","enrichments.geo.dip.latitude":"test latitude","enrichments.geo.sip.country":"test country","enrichments.geo.dip.city":"test city","enrichments.geo.sip.dmaCode":"test dmaCode","pkt":1,"enrichments.geo.sip.location_point":"test longitude,test latitude","ruflags":0,"roct":0,"sip":"216.21.170.221","rtag":0,"sp":80,"enrichments.geo.sip.longitude":"test longitude","enrichments.geo.sip.latitude":"test latitude","timestamp":1453994988562,"risn":0,"enrichments.host.dip.known_info.type":"printer","end_time":1453994988562,"enrichments.h
ost.dip.known_info.asset_value":"important","source.type":"yaf","rtt":"0.000"}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed b/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed
index 4b74794..86236ea 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed
+++ b/metron-streaming/Metron-Topologies/src/main/resources/SampleParsed/SnortParsed
@@ -1,3 +1,3 @@
-{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"22","ethsrc":"52:54:00:12:35:02","tcpseq":"0x9AFF3D7","dgmlen":"64","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0xC8761D52","original_string":"01\/27-16:01:04.877970 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,","icmpcode":"","tos":"0","id":"59677","timestamp":1453932941970,"ethdst":"08:00:27:7F:93:2D","src":"10.0.2.2","ttl":"64","source.type":"test","ethlen":"0x4E","iplen":"65536","icmptype":"","proto":"TCP","srcport":"56642","tcpflags":"***AP***","sig_id":"12","sig_generator":"129"}
-{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB45F7A","dgmlen":"96","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.612494 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,","icmpcode":"","tos":"0","id":"16785","timestamp":1456178820494,"ethdst":"08:00:27:7F:93:2D","src":"96.44.142.5","ttl":"64","source.type":"test","ethlen":"0x6E","iplen":"98304","icmptype":"","proto":"TCP","srcport":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129"}
-{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB508F2","dgmlen":"152","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.616775 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,","icmpcode":"","tos":"0","id":"16824","timestamp":1456178824775,"ethdst":"08:00:27:7F:93:2D","src":"96.44.142.5","ttl":"64","source.type":"test","ethlen":"0xA6","iplen":"155648","icmptype":"","proto":"TCP","srcport":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129"}
+{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"22","ethsrc":"52:54:00:12:35:02","tcpseq":"0x9AFF3D7","dgmlen":"64","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0xC8761D52","original_string":"01\/27-16:01:04.877970 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,10.0.2.2,56642,10.0.2.15,22,52:54:00:12:35:02,08:00:27:7F:93:2D,0x4E,***AP***,0x9AFF3D7,0xC8761D52,,0xFFFF,64,0,59677,64,65536,,,,","icmpcode":"","tos":"0","id":"59677","timestamp":1453932941970,"ethdst":"08:00:27:7F:93:2D","src":"10.0.2.2","ttl":"64","source.type":"test","ethlen":"0x4E","iplen":"65536","icmptype":"","proto":"TCP","srcport":"56642","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
+{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB45F7A","dgmlen":"96","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.612494 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0x6E,***AP***,0xDB45F7A,0x7701DD5B,,0xFFFF,64,0,16785,96,98304,,,,","icmpcode":"","tos":"0","id":"16785","timestamp":1456178820494,"ethdst":"08:00:27:7F:93:2D","src":"96.44.142.5","ttl":"64","source.type":"test","ethlen":"0x6E","iplen":"98304","icmptype":"","proto":"TCP","srcport":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
+{"msg":"\"Consecutive TCP small segments exceeding threshold\"","sig_rev":"1","dst":"10.0.2.15","dstport":"50895","ethsrc":"52:54:00:12:35:02","tcpseq":"0xDB508F2","dgmlen":"152","icmpid":"","tcplen":"","tcpwindow":"0xFFFF","icmpseq":"","tcpack":"0x7701DD5B","original_string":"02\/22-15:56:48.616775 ,129,12,1,\"Consecutive TCP small segments exceeding threshold\",TCP,96.44.142.5,80,10.0.2.15,50895,52:54:00:12:35:02,08:00:27:7F:93:2D,0xA6,***AP***,0xDB508F2,0x7701DD5B,,0xFFFF,64,0,16824,152,155648,,,,","icmpcode":"","tos":"0","id":"16824","timestamp":1456178824775,"ethdst":"08:00:27:7F:93:2D","src":"96.44.142.5","ttl":"64","source.type":"test","ethlen":"0xA6","iplen":"155648","icmptype":"","proto":"TCP","srcport":"80","tcpflags":"***AP***","sig_id":"12","sig_generator":"129", "is_alert" : "true"}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
index ef1318e..6e62e84 100644
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/EnrichmentIntegrationTest.java
@@ -17,7 +17,8 @@
*/
package org.apache.metron.integration;
-import com.google.common.base.Function;
+import com.google.common.base.*;
+import com.google.common.collect.Iterables;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.metron.Constants;
@@ -32,16 +33,19 @@ import org.apache.metron.integration.util.integration.ReadinessState;
import org.apache.metron.integration.util.integration.components.ElasticSearchComponent;
import org.apache.metron.integration.util.integration.components.FluxTopologyComponent;
import org.apache.metron.integration.util.integration.components.KafkaWithZKComponent;
+import org.apache.metron.integration.util.mock.MockGeoAdapter;
import org.apache.metron.integration.util.mock.MockHTable;
import org.apache.metron.integration.util.threatintel.ThreatIntelHelper;
import org.apache.metron.reference.lookup.LookupKV;
import org.apache.metron.utils.SourceConfigUtils;
import org.junit.Assert;
import org.junit.Test;
+import org.apache.metron.utils.JSONUtils;
import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
+import java.io.PrintWriter;
import java.io.Serializable;
import java.text.SimpleDateFormat;
import java.util.*;
@@ -66,7 +70,7 @@ public class EnrichmentIntegrationTest {
@Test
public void test() throws Exception {
final String dateFormat = "yyyy.MM.dd.hh";
- final String index = "yaf_" + new SimpleDateFormat(dateFormat).format(new Date());
+ final String index = "yaf_index_" + new SimpleDateFormat(dateFormat).format(new Date());
String yafConfig = "{\n" +
" \"index\": \"yaf\",\n" +
" \"batchSize\": 5,\n" +
@@ -142,7 +146,9 @@ public class EnrichmentIntegrationTest {
.withComponent("kafka", kafkaComponent)
.withComponent("elasticsearch", esComponent)
.withComponent("storm", fluxComponent)
- .withTimeBetweenAttempts(10000)
+ .withMillisecondsBetweenAttempts(10000)
+ .withNumRetries(30)
+ .withMaxTimeMS(300000)
.build();
runner.start();
fluxComponent.submitTopology();
@@ -154,7 +160,7 @@ public class EnrichmentIntegrationTest {
ElasticSearchComponent elasticSearchComponent = runner.getComponent("elasticsearch", ElasticSearchComponent.class);
if(elasticSearchComponent.hasIndex(index)) {
try {
- docs = elasticSearchComponent.getAllIndexedDocs(index, "yaf");
+ docs = elasticSearchComponent.getAllIndexedDocs(index, "yaf_doc");
} catch (IOException e) {
throw new IllegalStateException("Unable to retrieve indexed documents.", e);
}
@@ -177,19 +183,209 @@ public class EnrichmentIntegrationTest {
List<byte[]> sampleIndexedMessages = TestUtils.readSampleData(sampleIndexedPath);
Assert.assertEquals(sampleIndexedMessages.size(), docs.size());
- for (int i = 0; i < docs.size(); i++) {
- String doc = docs.get(i).toString();
- String sampleIndexedMessage = new String(sampleIndexedMessages.get(i));
- assertEqual(sampleIndexedMessage, doc);
+
+ for (Map<String, Object> doc : docs) {
+ baseValidation(doc);
+
+ hostEnrichmentValidation(doc);
+ geoEnrichmentValidation(doc);
+ threatIntelValidation(doc);
+
}
runner.stop();
}
- public static void assertEqual(String doc1, String doc2) {
- Assert.assertEquals(doc1.length(), doc2.length());
- char[] c1 = doc1.toCharArray();
- Arrays.sort(c1);
- char[] c2 = doc2.toCharArray();
- Arrays.sort(c2);
- Assert.assertArrayEquals(c1, c2);
+
+ public static void baseValidation(Map<String, Object> jsonDoc) {
+ assertEnrichmentsExists("threatintels.", setOf("ip"), jsonDoc.keySet());
+ assertEnrichmentsExists("enrichments.", setOf("geo", "host"), jsonDoc.keySet());
+ for(Map.Entry<String, Object> kv : jsonDoc.entrySet()) {
+ //ensure no values are empty.
+ Assert.assertTrue(kv.getValue().toString().length() > 0);
+ }
+ //ensure we always have a source ip and destination ip
+ Assert.assertNotNull(jsonDoc.get("sip"));
+ Assert.assertNotNull(jsonDoc.get("dip"));
+ }
+
+ private static class EvaluationPayload {
+ Map<String, Object> indexedDoc;
+ String key;
+ public EvaluationPayload(Map<String, Object> indexedDoc, String key) {
+ this.indexedDoc = indexedDoc;
+ this.key = key;
+ }
+ }
+
+ private static enum HostEnrichments implements Predicate<EvaluationPayload>{
+ LOCAL_LOCATION(new Predicate<EvaluationPayload>() {
+
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.local").equals("YES");
+ }
+ })
+ ,UNKNOWN_LOCATION(new Predicate<EvaluationPayload>() {
+
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.local").equals("UNKNOWN");
+ }
+ })
+ ,IMPORTANT(new Predicate<EvaluationPayload>() {
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.asset_value").equals("important");
+ }
+ })
+ ,PRINTER_TYPE(new Predicate<EvaluationPayload>() {
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.type").equals("printer");
+ }
+ })
+ ,WEBSERVER_TYPE(new Predicate<EvaluationPayload>() {
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.type").equals("webserver");
+ }
+ })
+ ,UNKNOWN_TYPE(new Predicate<EvaluationPayload>() {
+ @Override
+ public boolean apply(@Nullable EvaluationPayload evaluationPayload) {
+ return evaluationPayload.indexedDoc.get("enrichments.host." + evaluationPayload.key + ".known_info.type").equals("unknown");
+ }
+ })
+ ;
+
+ Predicate<EvaluationPayload> _predicate;
+ HostEnrichments(Predicate<EvaluationPayload> predicate) {
+ this._predicate = predicate;
+ }
+
+ public boolean apply(EvaluationPayload payload) {
+ return _predicate.apply(payload);
+ }
+
+ }
+
+ private static void assertEnrichmentsExists(String topLevel, Set<String> expectedEnrichments, Set<String> keys) {
+ for(String key : keys) {
+ if(key.startsWith(topLevel)) {
+ String secondLevel = Iterables.get(Splitter.on(".").split(key), 1);
+ String message = "Found an enrichment/threat intel (" + secondLevel + ") that I didn't expect (expected enrichments :"
+ + Joiner.on(",").join(expectedEnrichments) + "), but it was not there. If you've created a new"
+ + " enrichment, then please add a validation method to this unit test. Otherwise, it's a solid error"
+ + " and should be investigated.";
+ Assert.assertTrue( message, expectedEnrichments.contains(secondLevel));
+ }
+ }
}
+ private static void threatIntelValidation(Map<String, Object> indexedDoc) {
+ if(keyPatternExists("threatintels.", indexedDoc)) {
+ //if we have any threat intel messages, we want to tag is_alert to true
+ Assert.assertEquals(indexedDoc.get("is_alert"), "true");
+ }
+ else {
+ //For YAF this is the case, but if we do snort later on, this will be invalid.
+ Assert.assertNull(indexedDoc.get("is_alert"));
+ }
+ //ip threat intels
+ if(keyPatternExists("threatintels.ip.", indexedDoc)) {
+ if(indexedDoc.get("sip").equals("10.0.2.3")) {
+ Assert.assertEquals(indexedDoc.get("threatintels.ip.sip.ip_threat_intel"), "alert");
+ }
+ else if(indexedDoc.get("dip").equals("10.0.2.3")) {
+ Assert.assertEquals(indexedDoc.get("threatintels.ip.dip.ip_threat_intel"), "alert");
+ }
+ else {
+ Assert.fail("There was a threat intels that I did not expect.");
+ }
+ }
+
+ }
+
+ private static void geoEnrichmentValidation(Map<String, Object> indexedDoc) {
+ //should have geo enrichment on every message due to mock geo adapter
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.dip.location_point"), MockGeoAdapter.DEFAULT_LOCATION_POINT);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.sip.location_point"), MockGeoAdapter.DEFAULT_LOCATION_POINT);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.dip.longitude"), MockGeoAdapter.DEFAULT_LONGITUDE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.sip.longitude"), MockGeoAdapter.DEFAULT_LONGITUDE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.dip.city"), MockGeoAdapter.DEFAULT_CITY);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.sip.city"), MockGeoAdapter.DEFAULT_CITY);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.dip.latitude"), MockGeoAdapter.DEFAULT_LATITUDE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.sip.latitude"), MockGeoAdapter.DEFAULT_LATITUDE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.dip.country"), MockGeoAdapter.DEFAULT_COUNTRY);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.sip.country"), MockGeoAdapter.DEFAULT_COUNTRY);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.dip.dmaCode"), MockGeoAdapter.DEFAULT_DMACODE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.sip.dmaCode"), MockGeoAdapter.DEFAULT_DMACODE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.dip.postalCode"), MockGeoAdapter.DEFAULT_POSTAL_CODE);
+ Assert.assertEquals(indexedDoc.get("enrichments.geo.sip.postalCode"), MockGeoAdapter.DEFAULT_POSTAL_CODE);
+ }
+
+ private static void hostEnrichmentValidation(Map<String, Object> indexedDoc) {
+ boolean enriched = false;
+ //important local printers
+ {
+ Set<String> ips = setOf("10.0.2.15", "10.60.10.254");
+ if (ips.contains(indexedDoc.get("sip"))) {
+ //this is a local, important, printer
+ Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+ ,HostEnrichments.IMPORTANT
+ ,HostEnrichments.PRINTER_TYPE
+ ).apply(new EvaluationPayload(indexedDoc, "sip"))
+ );
+ enriched = true;
+ }
+ if (ips.contains(indexedDoc.get("dip"))) {
+ Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+ ,HostEnrichments.IMPORTANT
+ ,HostEnrichments.PRINTER_TYPE
+ ).apply(new EvaluationPayload(indexedDoc, "dip"))
+ );
+ enriched = true;
+ }
+ }
+ //important local webservers
+ {
+ Set<String> ips = setOf("10.1.128.236");
+ if (ips.contains(indexedDoc.get("sip"))) {
+ //this is a local, important, printer
+ Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+ ,HostEnrichments.IMPORTANT
+ ,HostEnrichments.WEBSERVER_TYPE
+ ).apply(new EvaluationPayload(indexedDoc, "sip"))
+ );
+ enriched = true;
+ }
+ if (ips.contains(indexedDoc.get("dip"))) {
+ Assert.assertTrue(Predicates.and(HostEnrichments.LOCAL_LOCATION
+ ,HostEnrichments.IMPORTANT
+ ,HostEnrichments.WEBSERVER_TYPE
+ ).apply(new EvaluationPayload(indexedDoc, "dip"))
+ );
+ enriched = true;
+ }
+ }
+ if(!enriched) {
+ Assert.assertFalse(keyPatternExists("enrichments.host", indexedDoc));
+ }
+ }
+
+
+ private static boolean keyPatternExists(String pattern, Map<String, Object> indexedObj) {
+ for(String k : indexedObj.keySet()) {
+ if(k.startsWith(pattern)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ private static Set<String> setOf(String... items) {
+ Set<String> ret = new HashSet<>();
+ for(String item : items) {
+ ret.add(item);
+ }
+ return ret;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java
index c55a069..80688b7 100644
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/ParserIntegrationTest.java
@@ -18,27 +18,14 @@
package org.apache.metron.integration;
import com.google.common.base.Function;
-import kafka.api.FetchRequest;
-import kafka.api.FetchRequestBuilder;
-import kafka.consumer.ConsumerIterator;
-import kafka.javaapi.FetchResponse;
-import kafka.javaapi.consumer.SimpleConsumer;
-import kafka.javaapi.producer.Producer;
-import kafka.message.MessageAndMetadata;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.metron.Constants;
import org.apache.metron.integration.util.TestUtils;
import org.apache.metron.integration.util.UnitTestHelper;
import org.apache.metron.integration.util.integration.ComponentRunner;
import org.apache.metron.integration.util.integration.Processor;
import org.apache.metron.integration.util.integration.ReadinessState;
-import org.apache.metron.integration.util.integration.components.ElasticSearchComponent;
import org.apache.metron.integration.util.integration.components.FluxTopologyComponent;
import org.apache.metron.integration.util.integration.components.KafkaWithZKComponent;
-import org.apache.metron.integration.util.integration.util.KafkaUtil;
-import org.apache.metron.spout.pcap.HDFSWriterCallback;
-import org.apache.metron.test.converters.HexStringConverter;
import org.apache.metron.utils.SourceConfigUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.junit.Assert;
@@ -95,7 +82,7 @@ public abstract class ParserIntegrationTest {
ComponentRunner runner = new ComponentRunner.Builder()
.withComponent("kafka", kafkaComponent)
.withComponent("storm", fluxComponent)
- .withTimeBetweenAttempts(5000)
+ .withMillisecondsBetweenAttempts(5000)
.build();
runner.start();
fluxComponent.submitTopology();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java
index 594700b..a3db041 100644
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/TestUtils.java
@@ -29,7 +29,6 @@ public class TestUtils {
BufferedReader br = new BufferedReader(new FileReader(samplePath));
List<byte[]> ret = new ArrayList<>();
for (String line = null; (line = br.readLine()) != null; ) {
- long ts = System.currentTimeMillis();
ret.add(line.getBytes());
}
br.close();
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockGeoAdapter.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockGeoAdapter.java b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockGeoAdapter.java
index 62ae618..ee71cda 100644
--- a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockGeoAdapter.java
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/util/mock/MockGeoAdapter.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.integration.util.mock;
+import com.google.common.base.Joiner;
import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
import org.json.simple.JSONObject;
@@ -25,6 +26,15 @@ import java.io.Serializable;
public class MockGeoAdapter implements EnrichmentAdapter<String>,
Serializable {
+ public static final String DEFAULT_LOC_ID = "1";
+ public static final String DEFAULT_COUNTRY = "test country";
+ public static final String DEFAULT_CITY = "test city";
+ public static final String DEFAULT_POSTAL_CODE = "test postalCode";
+ public static final String DEFAULT_LATITUDE = "test latitude";
+ public static final String DEFAULT_LONGITUDE = "test longitude";
+ public static final String DEFAULT_DMACODE= "test dmaCode";
+ public static final String DEFAULT_LOCATION_POINT= Joiner.on(',').join(DEFAULT_LONGITUDE, DEFAULT_LATITUDE);
+
@Override
public void logAccess(String value) {
@@ -32,14 +42,14 @@ public class MockGeoAdapter implements EnrichmentAdapter<String>,
public JSONObject enrich(String metadata) {
JSONObject enriched = new JSONObject();
- enriched.put("locID", "1");
- enriched.put("country", "test country");
- enriched.put("city", "test city");
- enriched.put("postalCode", "test postalCode");
- enriched.put("latitude", "test latitude");
- enriched.put("longitude", "test longitude");
- enriched.put("dmaCode", "test dmaCode");
- enriched.put("location_point", enriched.get("longitude") + "," + enriched.get("latitude"));
+ enriched.put("locID", DEFAULT_LOC_ID);
+ enriched.put("country", DEFAULT_COUNTRY);
+ enriched.put("city", DEFAULT_CITY);
+ enriched.put("postalCode", DEFAULT_POSTAL_CODE);
+ enriched.put("latitude", DEFAULT_LATITUDE);
+ enriched.put("longitude", DEFAULT_LONGITUDE);
+ enriched.put("dmaCode", DEFAULT_DMACODE);
+ enriched.put("location_point", DEFAULT_LOCATION_POINT);
return enriched;
}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 499e323..d8ac9d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
<exclude>**/.*</exclude>
<exclude>**/.*/**</exclude>
<exclude>**/*.seed</exclude>
- <exclude>**/*.iml</exclude>
+ <exclude>**/*.iml</exclude>
<exclude>**/ansible.cfg</exclude>
<exclude>site/**</exclude>
<exclude>metron-ui/lib/public/**</exclude>
@@ -54,7 +54,6 @@
<exclude>**/src/main/resources/Sample*/**</exclude>
<exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>**/files/opensoc-ui</exclude>
- <exclude>**/*.iml</exclude>
</excludes>
</configuration>
</plugin>