You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2018/08/23 21:58:53 UTC
[1/4] metron git commit: METRON-1707 Port Profiler to Spark
(nickwallen) closes apache/metron#1150
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1699-create-batch-profiler 6fb50a18d -> 3bfbf018a
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/pom.xml b/metron-analytics/pom.xml
index b6460ea..dc72f3b 100644
--- a/metron-analytics/pom.xml
+++ b/metron-analytics/pom.xml
@@ -46,6 +46,7 @@
<module>metron-profiler</module>
<module>metron-profiler-client</module>
<module>metron-profiler-common</module>
+ <module>metron-profiler-spark</module>
</modules>
<dependencies>
<dependency>
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java
index e2aa54d..82fbe2e 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResult.java
@@ -65,6 +65,10 @@ public class ProfileResult implements Serializable {
this.triageExpressions = new ProfileTriageExpressions();
}
+ public ProfileResult() {
+ // needed for serialization and Spark's code generator
+ }
+
public ProfileResultExpressions getProfileExpressions() {
return profileExpressions;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java
index 2cada01..464b246 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/profiler/ProfileResultExpressions.java
@@ -35,6 +35,10 @@ public class ProfileResultExpressions implements Serializable {
this.expression = expression;
}
+ public ProfileResultExpressions() {
+ // needed for serialization and Spark's code generator
+ }
+
@JsonValue
public String getExpression() {
return expression;
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9a8c442..6696ec7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -137,6 +137,7 @@
<global_reflections_version>0.9.10</global_reflections_version>
<global_checkstyle_version>8.0</global_checkstyle_version>
<global_log4j_core_version>2.1</global_log4j_core_version>
+ <global_spark_version>2.3.1</global_spark_version>
</properties>
<profiles>
[2/4] metron git commit: METRON-1707 Port Profiler to Spark
(nickwallen) closes apache/metron#1150
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/test/resources/telemetry.json
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/telemetry.json b/metron-analytics/metron-profiler-spark/src/test/resources/telemetry.json
new file mode 100644
index 0000000..4a324cf
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/telemetry.json
@@ -0,0 +1,100 @@
+{"adapter.threatinteladapter.end.ts":"1530978697769","qclass_name":"qclass-32769","bro_timestamp":"1530978687.836793","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978696551","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978696550","adapter.hostfromjsonlistadapter.end.ts":"1530978696606","trans_id":0,"adapter.geoadapter.begin.ts":"1530978696857","uid":"CGs8rS1rqhyXRRgA64","protocol":"dns","original_string":"DNS | AA:false qclass_name:qclass-32769 id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:CGs8rS1rqhyXRRgA64 RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:32769 ts:1530978687.836793 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978697808","enrichmentjoinbolt.joiner.ts":"1530978696932","adapter.hostfromjsonlistadapter.begin.ts":"1530978696606","threatintelsplitterbolt.splitter.begin.ts"
:"1530978696949","Z":0,"ip_src_addr":"192.168.66.1","qclass":32769,"timestamp":1530978687836,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696857","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978696952","adapter.threatinteladapter.begin.ts":"1530978697764","ip_src_port":5353,"proto":"udp","guid":"90751ce5-703d-4b9f-8c2d-8e5c42e72262"}
+{"adapter.threatinteladapter.end.ts":"1530978697772","bro_timestamp":"1530978687.77394","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696605","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978696605","adapter.hostfromjsonlistadapter.end.ts":"1530978696649","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978696857","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"CBJatv2DcsW8fow3Dg","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49186 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:CBJatv2DcsW8fow3Dg referrer:http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745 resp_mime_types:[\"text\\\/html\"] trans_depth:1 hos
t:r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in status_msg:OK id.orig_h:192.168.138.158 response_body_len:121635 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.77394 id.resp_h:62.75.195.236 resp_fuids:[\"F77a061yn9H0cUBGVa\"]","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978697808","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","enrichmentjoinbolt.joiner.ts":"1530978696943","adapter.hostfromjsonlistadapter.begin.ts":"1530978696607","threatintelsplitterbolt.splitter.begin.ts":"1530978696952","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["
F77a061yn9H0cUBGVa"],"timestamp":1530978687773,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696858","referrer":"http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745","threatintelsplitterbolt.splitter.end.ts":"1530978696952","adapter.threatinteladapter.begin.ts":"1530978697769","ip_src_port":49186,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"f5b315b0-e776-481a-9f28-765fdb19e6e8","response_body_len":121635}
+{"adapter.threatinteladapter.end.ts":"1530978697776","bro_timestamp":"1530978687.916811","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978696606","enrichmentsplitterbolt.splitter.begin.ts":"1530978696606","adapter.hostfromjsonlistadapter.end.ts":"1530978696650","adapter.geoadapter.begin.ts":"1530978696858","uid":"CUrRne3iLIxXavQtci","trans_depth":6,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters\/metron_cluster\/components\/?ServiceComponentInfo\/component_name=APP_TIMELINE_SERVER|ServiceComponentInfo\/category=MASTER&fields=ServiceComponentInfo\/service_name,host_components\/HostRoles\/display_name,host_components\/HostRoles\/host_name,host_components\/HostRoles\/state,host_components\/HostRoles\/maintenance_state,host_components\/HostRoles\/stale_configs,host_components\/HostRoles\/ha_state,host_components\/HostRoles\/desired_admin_state,,host_components\/metrics\/jvm\/memHeapUsedM,
host_components\/metrics\/jvm\/HeapMemoryMax,host_components\/metrics\/jvm\/HeapMemoryUsed,host_components\/metrics\/jvm\/memHeapCommittedM,host_components\/metrics\/mapred\/jobtracker\/trackers_decommissioned,host_components\/metrics\/cpu\/cpu_wio,host_components\/metrics\/rpc\/client\/RpcQueueTime_avg_time,host_components\/metrics\/dfs\/FSNamesystem\/*,host_components\/metrics\/dfs\/namenode\/Version,host_components\/metrics\/dfs\/namenode\/LiveNodes,host_components\/metrics\/dfs\/namenode\/DeadNodes,host_components\/metrics\/dfs\/namenode\/DecomNodes,host_components\/metrics\/dfs\/namenode\/TotalFiles,host_components\/metrics\/dfs\/namenode\/UpgradeFinalized,host_components\/metrics\/dfs\/namenode\/Safemode,host_components\/metrics\/runtime\/StartTime,host_components\/metrics\/hbase\/master\/IsActiveMaster,host_components\/metrics\/hbase\/master\/MasterStartTime,host_components\/metrics\/hbase\/master\/MasterActiveTime,host_components\/metrics\/hbase\/master\/AverageLoad,host_com
ponents\/metrics\/master\/AssignmentManger\/ritCount,metrics\/api\/v1\/cluster\/summary,metrics\/api\/v1\/topology\/summary,metrics\/api\/v1\/nimbus\/summary,host_components\/metrics\/yarn\/Queue,host_components\/metrics\/yarn\/ClusterMetrics\/NumActiveNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumLostNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumUnhealthyNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumRebootedNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumDecommissionedNMs&minimal_response=true&_=1484168361295 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:6 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978687.916811 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978697808","host":"node1","enrichmentjoinbolt.joiner.ts":"15309786
96948","adapter.hostfromjsonlistadapter.begin.ts":"1530978696649","threatintelsplitterbolt.splitter.begin.ts":"1530978696953","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978687916,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster\/components\/?ServiceComponentInfo\/component_name=APP_TIMELINE_SERVER|ServiceComponentInfo\/category=MASTER&fields=ServiceComponentInfo\/service_name,host_components\/HostRoles\/display_name,host_components\/HostRoles\/host_name,host_components\/HostRoles\/state,host_components\/HostRoles\/maintenance_state,host_components\/HostRoles\/stale_configs,host_components\/HostRoles\/ha_state,host_components\/HostRoles\/desired_admin_state,,host_components\/metrics\/jvm\/memHeapUsedM,host_components\/metrics\/jvm\/HeapMemoryMax,host_components\/metrics\/jvm\/HeapMemoryUsed,host_components\/metrics
\/jvm\/memHeapCommittedM,host_components\/metrics\/mapred\/jobtracker\/trackers_decommissioned,host_components\/metrics\/cpu\/cpu_wio,host_components\/metrics\/rpc\/client\/RpcQueueTime_avg_time,host_components\/metrics\/dfs\/FSNamesystem\/*,host_components\/metrics\/dfs\/namenode\/Version,host_components\/metrics\/dfs\/namenode\/LiveNodes,host_components\/metrics\/dfs\/namenode\/DeadNodes,host_components\/metrics\/dfs\/namenode\/DecomNodes,host_components\/metrics\/dfs\/namenode\/TotalFiles,host_components\/metrics\/dfs\/namenode\/UpgradeFinalized,host_components\/metrics\/dfs\/namenode\/Safemode,host_components\/metrics\/runtime\/StartTime,host_components\/metrics\/hbase\/master\/IsActiveMaster,host_components\/metrics\/hbase\/master\/MasterStartTime,host_components\/metrics\/hbase\/master\/MasterActiveTime,host_components\/metrics\/hbase\/master\/AverageLoad,host_components\/metrics\/master\/AssignmentManger\/ritCount,metrics\/api\/v1\/cluster\/summary,metrics\/api\/v1\/topology\
/summary,metrics\/api\/v1\/nimbus\/summary,host_components\/metrics\/yarn\/Queue,host_components\/metrics\/yarn\/ClusterMetrics\/NumActiveNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumLostNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumUnhealthyNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumRebootedNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumDecommissionedNMs&minimal_response=true&_=1484168361295","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696858","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697772","ip_src_port":50451,"guid":"db5e7329-9439-4a8a-972b-05d22d08e1fa","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978697777","bro_timestamp":"1530978687.073175","status_code":404,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696609","enrichments.geo.ip_dst_addr.city":"Phoenix","enrichments.geo.ip_dst_addr.latitude":"33.4499","enrichmentsplitterbolt.splitter.begin.ts":"1530978696609","adapter.hostfromjsonlistadapter.end.ts":"1530978696650","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5308655","adapter.geoadapter.begin.ts":"1530978696858","enrichments.geo.ip_dst_addr.postalCode":"85004","uid":"CxQY13LFLIWBK5kw6","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49203 status_code:404 method:POST request_body_len:110 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/twentyfifteen\/img5.php?f=ka6nnuvccqlw9 tags:[] uid:CxQY13LFLIWBK5kw6 resp_mime_types:[\"text\\\/html\"] trans_depth:1 orig_fuids:[\"FUF7cQ2NWtIJObUXFf\"] host:run
love.us status_msg:Not Found id.orig_h:192.168.138.158 response_body_len:357 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.073175 id.resp_h:204.152.254.221 resp_fuids:[\"FNXPE1PFFrR89EeJa\"]","ip_dst_addr":"204.152.254.221","threatinteljoinbolt.joiner.ts":"1530978697808","enrichments.geo.ip_dst_addr.dmaCode":"753","host":"runlove.us","enrichmentjoinbolt.joiner.ts":"1530978696948","adapter.hostfromjsonlistadapter.begin.ts":"1530978696650","threatintelsplitterbolt.splitter.begin.ts":"1530978696953","enrichments.geo.ip_dst_addr.longitude":"-112.0712","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FNXPE1PFFrR89EeJa"],"timestamp":1530978687073,"method":"POST","request_body_len"
:110,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/twentyfifteen\/img5.php?f=ka6nnuvccqlw9","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696870","threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697776","orig_fuids":["FUF7cQ2NWtIJObUXFf"],"ip_src_port":49203,"enrichments.geo.ip_dst_addr.location_point":"33.4499,-112.0712","status_msg":"Not Found","guid":"1d9eefeb-832b-4262-a800-5b67da9f7277","response_body_len":357}
+{"adapter.threatinteladapter.end.ts":"1530978697780","bro_timestamp":"1530978687.027914","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696609","enrichments.geo.ip_dst_addr.city":"Los Angeles","enrichments.geo.ip_dst_addr.latitude":"34.0494","enrichmentsplitterbolt.splitter.begin.ts":"1530978696609","adapter.hostfromjsonlistadapter.end.ts":"1530978696651","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5368361","adapter.geoadapter.begin.ts":"1530978696870","enrichments.geo.ip_dst_addr.postalCode":"90014","uid":"CxZIVD4f5vBwpXUjwf","resp_mime_types":["text\/plain"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49198 status_code:200 method:POST request_body_len:134 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/grizzly\/img5.php?c=cdcnw7cfz43rmtg tags:[] uid:CxZIVD4f5vBwpXUjwf resp_mime_types:[\"text\\\/plain\"] trans_depth:1 orig_fuids:[\"FiPZ8g4gdpjEyHuez2\"] host
:comarksecurity.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:14 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.027914 id.resp_h:72.34.49.86 resp_fuids:[\"FM8l2i6ib3vOd45ob\"]","ip_dst_addr":"72.34.49.86","threatinteljoinbolt.joiner.ts":"1530978697808","enrichments.geo.ip_dst_addr.dmaCode":"803","host":"comarksecurity.com","enrichmentjoinbolt.joiner.ts":"1530978696949","adapter.hostfromjsonlistadapter.begin.ts":"1530978696650","threatintelsplitterbolt.splitter.begin.ts":"1530978696953","enrichments.geo.ip_dst_addr.longitude":"-118.2641","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FM8l2i6ib3vOd45ob"],"timestamp":1530978687027,"method":"POST","request_body_
len":134,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/grizzly\/img5.php?c=cdcnw7cfz43rmtg","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696875","threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697778","orig_fuids":["FiPZ8g4gdpjEyHuez2"],"ip_src_port":49198,"enrichments.geo.ip_dst_addr.location_point":"34.0494,-118.2641","status_msg":"OK","guid":"0c21f313-5cb7-46de-b62a-b429c565bfb0","response_body_len":14}
+{"adapter.threatinteladapter.end.ts":"1530978697782","bro_timestamp":"1530978687.58302","ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696609","enrichments.geo.ip_dst_addr.latitude":"48.8582","enrichmentsplitterbolt.splitter.begin.ts":"1530978696609","adapter.hostfromjsonlistadapter.end.ts":"1530978696651","enrichments.geo.ip_dst_addr.country":"FR","adapter.geoadapter.begin.ts":"1530978696875","uid":"CT2ax04BCxPW20AlGc","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49195 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:CT2ax04BCxPW20AlGc trans_depth:1 host:ip-addr.es id.orig_h:192.168.138.158 response_body_len:0 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.58302 id.resp_h:188.165.164.184","ip_dst_addr":"188.165.164.184","threatinteljoinbolt.joiner.ts":"1530978697809","host":"ip-addr.es","e
nrichmentjoinbolt.joiner.ts":"1530978696949","adapter.hostfromjsonlistadapter.begin.ts":"1530978696651","threatintelsplitterbolt.splitter.begin.ts":"1530978696953","enrichments.geo.ip_dst_addr.longitude":"2.3387000000000002","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","timestamp":1530978687583,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696879","threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697780","ip_src_port":49195,"enrichments.geo.ip_dst_addr.location_point":"48.8582,2.3387000000000002","guid":"ed0d58c1-88cb-4f4e-ab7b-ee1a36e7cdcb","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978697783","qclass_name":"C_INTERNET","bro_timestamp":"1530978687.445971","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978696610","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978696610","adapter.hostfromjsonlistadapter.end.ts":"1530978696651","trans_id":0,"adapter.geoadapter.begin.ts":"1530978696879","uid":"ChMDrL20pLP4UzCncj","protocol":"dns","original_string":"DNS | AA:false qclass_name:C_INTERNET id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:ChMDrL20pLP4UzCncj RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:1 ts:1530978687.445971 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978697809","enrichmentjoinbolt.joiner.ts":"1530978696949","adapter.hostfromjsonlistadapter.begin.ts":"1530978696651","threatintelsplitterbolt.splitter.begin.ts":"153097
8696953","Z":0,"ip_src_addr":"192.168.66.1","qclass":1,"timestamp":1530978687445,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696879","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978696953","adapter.threatinteladapter.begin.ts":"1530978697783","ip_src_port":5353,"proto":"udp","guid":"a6f4fe3a-c485-4521-bcfe-b2600746885e"}
+{"TTLs":[21599.0],"adapter.threatinteladapter.end.ts":"1530978697784","qclass_name":"C_INTERNET","bro_timestamp":"1530978687.053752","qtype_name":"A","ip_dst_port":53,"enrichmentsplitterbolt.splitter.end.ts":"1530978696610","qtype":1,"rejected":false,"answers":["188.165.164.184"],"enrichmentsplitterbolt.splitter.begin.ts":"1530978696610","adapter.hostfromjsonlistadapter.end.ts":"1530978696652","trans_id":15553,"adapter.geoadapter.begin.ts":"1530978696879","uid":"CoiTkw2sb9stNr10zg","protocol":"dns","original_string":"DNS | AA:false TTLs:[21599.0] qclass_name:C_INTERNET id.orig_p:53571 qtype_name:A qtype:1 rejected:false id.resp_p:53 query:ip-addr.es answers:[\"188.165.164.184\"] trans_id:15553 rcode:0 rcode_name:NOERROR TC:false RA:true uid:CoiTkw2sb9stNr10zg RD:true proto:udp id.orig_h:192.168.138.158 Z:0 qclass:1 ts:1530978687.053752 id.resp_h:192.168.138.2","ip_dst_addr":"192.168.138.2","threatinteljoinbolt.joiner.ts":"1530978697809","enrichmentjoinbolt.joiner.ts":"1530978696953"
,"adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"1530978696961","Z":0,"ip_src_addr":"192.168.138.158","qclass":1,"timestamp":1530978687053,"AA":false,"query":"ip-addr.es","rcode":0,"rcode_name":"NOERROR","TC":false,"RA":true,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696879","RD":true,"threatintelsplitterbolt.splitter.end.ts":"1530978696961","adapter.threatinteladapter.begin.ts":"1530978697783","ip_src_port":53571,"proto":"udp","guid":"bbfd5e54-db09-455e-b01f-b6cbbd444e88"}
+{"adapter.threatinteladapter.end.ts":"1530978697784","bro_timestamp":"1530978687.267256","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696610","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978696610","adapter.hostfromjsonlistadapter.end.ts":"1530978696652","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978696880","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"CID7qb45BoqLfAMHic","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49193 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/?34eaf8bd50d85d8c6baacb45f0a7b22e tags:[] uid:CID7qb45BoqLfAMHic trans_depth:1 host:62.75.195.236 status_msg:OK id.orig_h:192.168.138.158 response_body_len:0 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .N
ET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.267256 id.resp_h:62.75.195.236","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978697809","host":"62.75.195.236","enrichmentjoinbolt.joiner.ts":"1530978696953","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"1530978696961","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","timestamp":1530978687267,"method":"GET","request_body_len":0,"uri":"\/?34eaf8bd50d85d8c6baacb45f0a7b22e","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696880","threatintelsplitterbolt.splitter.end.ts":"1530978696961","adapter.threatinteladapter.begin.ts":"1530978697784","ip_src_port":49193,"enrichments.geo.ip_dst_addr.locatio
n_point":"48.5839,7.7455","status_msg":"OK","guid":"ad2f6714-2a4a-4262-8ce0-1940f3e8f340","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978697786","bro_timestamp":"1530978687.417086","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696610","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978696610","adapter.hostfromjsonlistadapter.end.ts":"1530978696652","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978696880","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CEkDUW1JYqnTIkYzc1","resp_mime_types":["image\/png"],"trans_depth":2,"protocol":"http","original_string":"HTTP | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/lb.png tags:[] uid:CEkDUW1JYqnTIkYzc1 referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:2 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:192.1
68.138.158 response_body_len:239 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978687.417086 id.resp_h:95.163.121.204 resp_fuids:[\"FZy6Lx4RGFmQ1AZeU8\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978697809","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978696953","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"1530978696961","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FZy6Lx4RGFmQ1AZeU8"],"timestamp":1530978687417,"method":"GET","request_body_len":0,"uri":"\/img\/lb.png","tags":[],"source.type":"bro","adapter.geoada
pter.end.ts":"1530978696887","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978696961","adapter.threatinteladapter.begin.ts":"1530978697784","ip_src_port":49210,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"9711e04e-9926-4606-a7e8-e719dac535e6","response_body_len":239}
+{"adapter.threatinteladapter.end.ts":"1530978697786","qclass_name":"C_INTERNET","bro_timestamp":"1530978694.884106","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696652","trans_id":0,"adapter.geoadapter.begin.ts":"1530978696887","uid":"CkwtUK1ANyyZwj0PW1","protocol":"dns","original_string":"DNS | AA:false qclass_name:C_INTERNET id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:CkwtUK1ANyyZwj0PW1 RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:1 ts:1530978694.884106 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978697809","enrichmentjoinbolt.joiner.ts":"1530978696953","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"153097
8696961","Z":0,"ip_src_addr":"192.168.66.1","qclass":1,"timestamp":1530978694884,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978696961","adapter.threatinteladapter.begin.ts":"1530978697786","ip_src_port":5353,"proto":"udp","guid":"f9c14d84-59c5-4598-97b7-5d6e95aba4e6"}
+{"adapter.threatinteladapter.end.ts":"1530978697786","bro_timestamp":"1530978694.621046","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696653","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978696887","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"C1ia4w3K4ngOWPmAsi","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49186 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:C1ia4w3K4ngOWPmAsi referrer:http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745 resp_mime_types:[\"text\\\/html\"] trans_depth:1 ho
st:r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in status_msg:OK id.orig_h:192.168.138.158 response_body_len:121635 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.621046 id.resp_h:62.75.195.236 resp_fuids:[\"FKGK2W1X8Bfk7D7XD9\"]","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978697809","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","enrichmentjoinbolt.joiner.ts":"1530978696953","adapter.hostfromjsonlistadapter.begin.ts":"1530978696652","threatintelsplitterbolt.splitter.begin.ts":"1530978696962","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":
["FKGK2W1X8Bfk7D7XD9"],"timestamp":1530978694621,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","referrer":"http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745","threatintelsplitterbolt.splitter.end.ts":"1530978696962","adapter.threatinteladapter.begin.ts":"1530978697786","ip_src_port":49186,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"98769db4-ee20-4f69-bb04-2e7005de9c6d","response_body_len":121635}
+{"adapter.threatinteladapter.end.ts":"1530978697787","bro_timestamp":"1530978694.641679","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696653","adapter.geoadapter.begin.ts":"1530978696887","uid":"CUrRne3iLIxXavQtci","trans_depth":254,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/persist\/wizard-data?_=1484169473684 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:254 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978694.641679 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978697810","host":"node1","enrichmentjoinbolt.joiner.ts":"1530978696954","adapter.hos
tfromjsonlistadapter.begin.ts":"1530978696653","threatintelsplitterbolt.splitter.begin.ts":"1530978696962","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978694641,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/persist\/wizard-data?_=1484169473684","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978696962","adapter.threatinteladapter.begin.ts":"1530978697787","ip_src_port":50451,"guid":"ffda80d8-44e3-42db-b72a-d2fa7cf38042","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978697788","bro_timestamp":"1530978694.388009","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696653","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978696887","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CJyQ1119VSEe7SGiTa","resp_mime_types":["image\/x-icon"],"trans_depth":2,"protocol":"http","original_string":"HTTP | id.orig_p:49207 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/favicon.ico tags:[] uid:CJyQ1119VSEe7SGiTa resp_mime_types:[\"image\\\/x-icon\"] trans_depth:2 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:318 user_agent:Mozilla\/
4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.388009 id.resp_h:95.163.121.204 resp_fuids:[\"F0Pvmv1dj2gRa9c7v\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978697810","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978696954","adapter.hostfromjsonlistadapter.begin.ts":"1530978696653","threatintelsplitterbolt.splitter.begin.ts":"1530978696962","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["F0Pvmv1dj2gRa9c7v"],"timestamp":1530978694388,"method":"GET","request_body_len":0,"uri":"\/favicon.ico","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","threatintelsplitterbolt.s
plitter.end.ts":"1530978696962","adapter.threatinteladapter.begin.ts":"1530978697787","ip_src_port":49207,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"66eda80b-7f24-4aec-85b9-e381e128dfc7","response_body_len":318}
+{"adapter.threatinteladapter.end.ts":"1530978697788","bro_timestamp":"1530978694.979947","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978696611","enrichments.geo.ip_dst_addr.city":"Los Angeles","enrichments.geo.ip_dst_addr.latitude":"34.0494","enrichmentsplitterbolt.splitter.begin.ts":"1530978696611","adapter.hostfromjsonlistadapter.end.ts":"1530978696653","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5368361","adapter.geoadapter.begin.ts":"1530978696887","enrichments.geo.ip_dst_addr.postalCode":"90014","uid":"COZAhy4ljJ4lBc5bgf","resp_mime_types":["text\/plain"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49204 status_code:200 method:POST request_body_len:110 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/grizzly\/img5.php?u=ka6nnuvccqlw9 tags:[] uid:COZAhy4ljJ4lBc5bgf resp_mime_types:[\"text\\\/plain\"] trans_depth:1 orig_fuids:[\"FgncKy2eauwZjDL6h9\"] host:c
omarksecurity.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:14 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.979947 id.resp_h:72.34.49.86 resp_fuids:[\"FDVxtiyWLP0KeNRg8\"]","ip_dst_addr":"72.34.49.86","threatinteljoinbolt.joiner.ts":"1530978697810","enrichments.geo.ip_dst_addr.dmaCode":"803","host":"comarksecurity.com","enrichmentjoinbolt.joiner.ts":"1530978696954","adapter.hostfromjsonlistadapter.begin.ts":"1530978696653","threatintelsplitterbolt.splitter.begin.ts":"1530978696962","enrichments.geo.ip_dst_addr.longitude":"-118.2641","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FDVxtiyWLP0KeNRg8"],"timestamp":1530978694979,"method":"POST","request_body_le
n":110,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/grizzly\/img5.php?u=ka6nnuvccqlw9","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978696887","threatintelsplitterbolt.splitter.end.ts":"1530978696962","adapter.threatinteladapter.begin.ts":"1530978697788","orig_fuids":["FgncKy2eauwZjDL6h9"],"ip_src_port":49204,"enrichments.geo.ip_dst_addr.location_point":"34.0494,-118.2641","status_msg":"OK","guid":"a5bc5b67-a861-43e2-9232-fb902239cea3","response_body_len":14}
+{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.045879","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702588","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702588","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CmNSa535EEM4iN5uwh","resp_mime_types":["image\/png"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49209 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/flags\/de.png tags:[] uid:CmNSa535EEM4iN5uwh referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:1 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_
h:192.168.138.158 response_body_len:534 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.045879 id.resp_h:95.163.121.204 resp_fuids:[\"FZgahz2hSOfoAP9y1l\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702609","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702598","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FZgahz2hSOfoAP9y1l"],"timestamp":1530978694045,"method":"GET","request_body_len":0,"uri":"\/img\/flags\/de.png","tags":[],"source.type":"bro","
adapter.geoadapter.end.ts":"1530978702593","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","ip_src_port":49209,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"9e19e186-6aba-45ad-8b70-9e696ef02448","response_body_len":534}
+{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.98983","status_code":404,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702589","enrichments.geo.ip_dst_addr.city":"Phoenix","enrichments.geo.ip_dst_addr.latitude":"33.4499","enrichmentsplitterbolt.splitter.begin.ts":"1530978702589","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5308655","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"85004","uid":"CPbKPD2f2Vg9rvtXXk","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49199 status_code:404 method:POST request_body_len:96 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/twentyfifteen\/img5.php?l=8r1gf1b2t1kuq42 tags:[] uid:CPbKPD2f2Vg9rvtXXk resp_mime_types:[\"text\\\/html\"] trans_depth:1 orig_fuids:[\"FVYpPq1KmqTn8vOfT\"] host:ru
nlove.us status_msg:Not Found id.orig_h:192.168.138.158 response_body_len:357 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.98983 id.resp_h:204.152.254.221 resp_fuids:[\"FnmQJXPDEKwZZ8TMf\"]","ip_dst_addr":"204.152.254.221","threatinteljoinbolt.joiner.ts":"1530978702609","enrichments.geo.ip_dst_addr.dmaCode":"753","host":"runlove.us","enrichmentjoinbolt.joiner.ts":"1530978702598","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"-112.0712","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FnmQJXPDEKwZZ8TMf"],"timestamp":1530978694989,"method":"POST","request_body_len"
:96,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/twentyfifteen\/img5.php?l=8r1gf1b2t1kuq42","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702593","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","orig_fuids":["FVYpPq1KmqTn8vOfT"],"ip_src_port":49199,"enrichments.geo.ip_dst_addr.location_point":"33.4499,-112.0712","status_msg":"Not Found","guid":"23070f86-2358-4f4c-9bf4-a612afc8c3e3","response_body_len":357}
+{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.665931","status_code":404,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702589","enrichments.geo.ip_dst_addr.city":"Phoenix","enrichments.geo.ip_dst_addr.latitude":"33.4499","enrichmentsplitterbolt.splitter.begin.ts":"1530978702589","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5308655","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"85004","uid":"CQPUy829Fo1TwbqZh5","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49203 status_code:404 method:POST request_body_len:110 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/twentyfifteen\/img5.php?f=ka6nnuvccqlw9 tags:[] uid:CQPUy829Fo1TwbqZh5 resp_mime_types:[\"text\\\/html\"] trans_depth:1 orig_fuids:[\"FHf5Gv2fxGeTgj5aLk\"] host:r
unlove.us status_msg:Not Found id.orig_h:192.168.138.158 response_body_len:357 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.665931 id.resp_h:204.152.254.221 resp_fuids:[\"FuBgoE4Ro7nr1s5NO8\"]","ip_dst_addr":"204.152.254.221","threatinteljoinbolt.joiner.ts":"1530978702609","enrichments.geo.ip_dst_addr.dmaCode":"753","host":"runlove.us","enrichmentjoinbolt.joiner.ts":"1530978702599","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"-112.0712","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FuBgoE4Ro7nr1s5NO8"],"timestamp":1530978694665,"method":"POST","request_body_
len":110,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/twentyfifteen\/img5.php?f=ka6nnuvccqlw9","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702593","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","orig_fuids":["FHf5Gv2fxGeTgj5aLk"],"ip_src_port":49203,"enrichments.geo.ip_dst_addr.location_point":"33.4499,-112.0712","status_msg":"Not Found","guid":"41e087a9-84a3-41a2-af03-b0ade87ffa76","response_body_len":357}
+{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.939958","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702589","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CA0G2ASkF1efFirs7","resp_mime_types":["image\/png"],"trans_depth":2,"protocol":"http","original_string":"HTTP | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/lb.png tags:[] uid:CA0G2ASkF1efFirs7 referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:2 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:192.168
.138.158 response_body_len:239 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.939958 id.resp_h:95.163.121.204 resp_fuids:[\"FXqalu3YBvkNyrelff\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702610","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702599","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FXqalu3YBvkNyrelff"],"timestamp":1530978694939,"method":"GET","request_body_len":0,"uri":"\/img\/lb.png","tags":[],"source.type":"bro","adapter.geoadapt
er.end.ts":"1530978702593","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","ip_src_port":49210,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"b4a27884-579e-4266-b1d5-0c12f941924a","response_body_len":239}
+{"adapter.threatinteladapter.end.ts":"1530978702605","bro_timestamp":"1530978694.291127","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702593","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702593","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CodIOCgeqZXqVSCg6","resp_mime_types":["image\/png"],"trans_depth":4,"protocol":"http","original_string":"HTTP | id.orig_p:49205 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/bitcoin.png tags:[] uid:CodIOCgeqZXqVSCg6 referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:4 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:19
2.168.138.158 response_body_len:5523 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978694.291127 id.resp_h:95.163.121.204 resp_fuids:[\"Ft8inr3vk76ny20gZ2\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702610","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702599","adapter.hostfromjsonlistadapter.begin.ts":"1530978702593","threatintelsplitterbolt.splitter.begin.ts":"1530978702601","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["Ft8inr3vk76ny20gZ2"],"timestamp":1530978694291,"method":"GET","request_body_len":0,"uri":"\/img\/bitcoin.png","tags":[],"source.type":"bro","adapt
er.geoadapter.end.ts":"1530978702593","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702601","adapter.threatinteladapter.begin.ts":"1530978702605","ip_src_port":49205,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"105bf657-f1ec-4276-bfcd-091905599296","response_body_len":5523}
+{"adapter.threatinteladapter.end.ts":"1530978702609","bro_timestamp":"1530978698.168044","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978702597","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"ChNCWL3i4gNIYPkoDe","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49194 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/?60dbe33b908e0086292196ef001816bc tags:[] uid:ChNCWL3i4gNIYPkoDe trans_depth:1 host:62.75.195.236 status_msg:OK id.orig_h:192.168.138.158 response_body_len:0 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .N
ET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.168044 id.resp_h:62.75.195.236","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978702612","host":"62.75.195.236","enrichmentjoinbolt.joiner.ts":"1530978702601","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702605","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","timestamp":1530978698168,"method":"GET","request_body_len":0,"uri":"\/?60dbe33b908e0086292196ef001816bc","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702597","threatintelsplitterbolt.splitter.end.ts":"1530978702605","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":49194,"enrichments.geo.ip_dst_addr.locatio
n_point":"48.5839,7.7455","status_msg":"OK","guid":"5bd73342-6081-4de8-af0d-b68efab3bf95","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978702609","bro_timestamp":"1530978698.840044","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702597","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CX5zuR35fzQMB5VJmd","resp_mime_types":["text\/html"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49205 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/11iQmfg tags:[] uid:CX5zuR35fzQMB5VJmd resp_mime_types:[\"text\\\/html\"] trans_depth:1 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:3289 user_agent:Mozilla\/4.0 (comp
atible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.840044 id.resp_h:95.163.121.204 resp_fuids:[\"FjwIxc3D3tcVPcqmGc\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702612","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702601","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702605","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FjwIxc3D3tcVPcqmGc"],"timestamp":1530978698840,"method":"GET","request_body_len":0,"uri":"\/11iQmfg","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702597","threatintelsplitterbolt.splitter.end
.ts":"1530978702605","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":49205,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"f9dbf04b-bcc8-48a9-b858-8ca45b6f8274","response_body_len":3289}
+{"adapter.threatinteladapter.end.ts":"1530978702609","bro_timestamp":"1530978698.949395","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"C8Ljn32fwV1v4G45R8","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49188 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/aa25f5fe2875e3d0a244e6969e589cc4 tags:[] uid:C8Ljn32fwV1v4G45R8 trans_depth:1 host:62.75.195.236 status_msg:OK id.orig_h:192.168.138.158 response_body_len:861 ts:1530978698.949395 id.resp_h:62.75.195.236 resp_fuids:[\"FfQLue1qc3s7ZfGzH5\"]","ip_dst_ad
dr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978702612","host":"62.75.195.236","enrichmentjoinbolt.joiner.ts":"1530978702601","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702605","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","resp_fuids":["FfQLue1qc3s7ZfGzH5"],"timestamp":1530978698949,"method":"GET","request_body_len":0,"uri":"\/aa25f5fe2875e3d0a244e6969e589cc4","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702598","threatintelsplitterbolt.splitter.end.ts":"1530978702605","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":49188,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"e8cff10e-d3ab-4578-8aeb-f94c614b5bd6","response_body_len":861}
+{"adapter.threatinteladapter.end.ts":"1530978702609","qclass_name":"qclass-32769","bro_timestamp":"1530978698.075525","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","trans_id":0,"adapter.geoadapter.begin.ts":"1530978702598","uid":"C8DgGj1pj2jXyhi9g1","protocol":"dns","original_string":"DNS | AA:false qclass_name:qclass-32769 id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:C8DgGj1pj2jXyhi9g1 RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:32769 ts:1530978698.075525 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978702612","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts"
:"1530978702605","Z":0,"ip_src_addr":"192.168.66.1","qclass":32769,"timestamp":1530978698075,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978702598","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978702606","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":5353,"proto":"udp","guid":"f88c60ba-4062-411f-ae82-c9a86e0a0d1b"}
+{"adapter.threatinteladapter.end.ts":"1530978702609","bro_timestamp":"1530978698.312623","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702590","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978702590","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"CCTaln3ggV4dOqGETi","trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49194 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/?60dbe33b908e0086292196ef001816bc tags:[] uid:CCTaln3ggV4dOqGETi trans_depth:1 host:62.75.195.236 status_msg:OK id.orig_h:192.168.138.158 response_body_len:0 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .N
ET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.312623 id.resp_h:62.75.195.236","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978702612","host":"62.75.195.236","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702606","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","timestamp":1530978698312,"method":"GET","request_body_len":0,"uri":"\/?60dbe33b908e0086292196ef001816bc","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702598","threatintelsplitterbolt.splitter.end.ts":"1530978702606","adapter.threatinteladapter.begin.ts":"1530978702609","ip_src_port":49194,"enrichments.geo.ip_dst_addr.locatio
n_point":"48.5839,7.7455","status_msg":"OK","guid":"c0049477-bdc9-42ab-88fe-c088a7d9e76d","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.907146","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"Cnd9EM1uTP3PbJ0BS","resp_mime_types":["image\/png"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49209 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/flags\/de.png tags:[] uid:Cnd9EM1uTP3PbJ0BS referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:1 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:
192.168.138.158 response_body_len:534 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.907146 id.resp_h:95.163.121.204 resp_fuids:[\"Fck5px3MJLpCrDeCZ3\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702613","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702607","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["Fck5px3MJLpCrDeCZ3"],"timestamp":1530978698907,"method":"GET","request_body_len":0,"uri":"\/img\/flags\/de.png","tags":[],"source.type":"bro","ad
apter.geoadapter.end.ts":"1530978702598","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702607","adapter.threatinteladapter.begin.ts":"1530978702611","ip_src_port":49209,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"358e4eca-0f08-4c10-a307-881009c223b0","response_body_len":534}
+{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.884865","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"CJY1nx4uy46hVP4kmg","resp_mime_types":["text\/plain"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49206 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/style.css tags:[] uid:CJY1nx4uy46hVP4kmg referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"text\\\/plain\"] trans_depth:1 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:
192.168.138.158 response_body_len:4492 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.884865 id.resp_h:95.163.121.204 resp_fuids:[\"FindPO2TsX283BvQw3\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702613","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702607","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FindPO2TsX283BvQw3"],"timestamp":1530978698884,"method":"GET","request_body_len":0,"uri":"\/img\/style.css","tags":[],"source.type":"bro","adapt
er.geoadapter.end.ts":"1530978702598","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702607","adapter.threatinteladapter.begin.ts":"1530978702611","ip_src_port":49206,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"94df9cde-8877-43bf-97a6-d2e0bbc840c4","response_body_len":4492}
+{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.521985","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"C1qlzE2SalKbpWSJGi","resp_mime_types":["image\/png"],"trans_depth":3,"protocol":"http","original_string":"HTTP | id.orig_p:49210 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/button_pay.png tags:[] uid:C1qlzE2SalKbpWSJGi referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:3 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig
_h:192.168.138.158 response_body_len:727 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.521985 id.resp_h:95.163.121.204 resp_fuids:[\"Fd2ecB4nK7EKV7lLA1\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702613","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702607","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["Fd2ecB4nK7EKV7lLA1"],"timestamp":1530978698521,"method":"GET","request_body_len":0,"uri":"\/img\/button_pay.png","tags":[],"source.type":"bro"
,"adapter.geoadapter.end.ts":"1530978702598","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702607","adapter.threatinteladapter.begin.ts":"1530978702611","ip_src_port":49210,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"024a7ece-fce2-4ec2-86ee-e5e7d0dc2a5d","response_body_len":727}
+{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.077529","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"C5UfKV32U65H7ojqJd","resp_mime_types":["image\/png"],"trans_depth":4,"protocol":"http","original_string":"HTTP | id.orig_p:49205 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/bitcoin.png tags:[] uid:C5UfKV32U65H7ojqJd referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:4 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_h:
192.168.138.158 response_body_len:5523 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.077529 id.resp_h:95.163.121.204 resp_fuids:[\"Fy6w2R347d11rin2hg\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978702613","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978702602","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702607","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["Fy6w2R347d11rin2hg"],"timestamp":1530978698077,"method":"GET","request_body_len":0,"uri":"\/img\/bitcoin.png","tags":[],"source.type":"bro","ada
pter.geoadapter.end.ts":"1530978702598","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978702607","adapter.threatinteladapter.begin.ts":"1530978702611","ip_src_port":49205,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"626f9f8b-2af5-4c9f-a36a-3cfedea5614e","response_body_len":5523}
+{"adapter.threatinteladapter.end.ts":"1530978702611","bro_timestamp":"1530978698.241724","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978702591","enrichments.geo.ip_dst_addr.city":"Los Angeles","enrichments.geo.ip_dst_addr.latitude":"34.0494","enrichmentsplitterbolt.splitter.begin.ts":"1530978702591","adapter.hostfromjsonlistadapter.end.ts":"1530978702597","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5368361","adapter.geoadapter.begin.ts":"1530978702598","enrichments.geo.ip_dst_addr.postalCode":"90014","uid":"CJNiGM3zcyXHHORzFb","resp_mime_types":["text\/plain"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49198 status_code:200 method:POST request_body_len:134 id.resp_p:80 orig_mime_types:[\"text\\\/plain\"] uri:\/wp-content\/themes\/grizzly\/img5.php?c=cdcnw7cfz43rmtg tags:[] uid:CJNiGM3zcyXHHORzFb resp_mime_types:[\"text\\\/plain\"] trans_depth:1 orig_fuids:[\"FJWjcF3Z0qYg56Pw65\"] host
:comarksecurity.com status_msg:OK id.orig_h:192.168.138.158 response_body_len:14 user_agent:Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978698.241724 id.resp_h:72.34.49.86 resp_fuids:[\"FrVBRXxij8xG1u239\"]","ip_dst_addr":"72.34.49.86","threatinteljoinbolt.joiner.ts":"1530978702614","enrichments.geo.ip_dst_addr.dmaCode":"803","host":"comarksecurity.com","enrichmentjoinbolt.joiner.ts":"1530978702603","adapter.hostfromjsonlistadapter.begin.ts":"1530978702597","threatintelsplitterbolt.splitter.begin.ts":"1530978702608","enrichments.geo.ip_dst_addr.longitude":"-118.2641","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FrVBRXxij8xG1u239"],"timestamp":1530978698241,"method":"POST","request_body_
len":134,"orig_mime_types":["text\/plain"],"uri":"\/wp-content\/themes\/grizzly\/img5.php?c=cdcnw7cfz43rmtg","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978702598","threatintelsplitterbolt.splitter.end.ts":"1530978702608","adapter.threatinteladapter.begin.ts":"1530978702611","orig_fuids":["FJWjcF3Z0qYg56Pw65"],"ip_src_port":49198,"enrichments.geo.ip_dst_addr.location_point":"34.0494,-118.2641","status_msg":"OK","guid":"40b1b0b6-a51c-41a2-9d97-ef26badb79fa","response_body_len":14}
+{"adapter.threatinteladapter.end.ts":"1530978710497","bro_timestamp":"1530978704.958145","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978710473","enrichments.geo.ip_dst_addr.city":"Strasbourg","enrichments.geo.ip_dst_addr.latitude":"48.5839","enrichmentsplitterbolt.splitter.begin.ts":"1530978710473","adapter.hostfromjsonlistadapter.end.ts":"1530978710476","enrichments.geo.ip_dst_addr.country":"FR","enrichments.geo.ip_dst_addr.locID":"2973783","adapter.geoadapter.begin.ts":"1530978710476","enrichments.geo.ip_dst_addr.postalCode":"67100","uid":"C7KeXZ1jvzj9qkSqt7","resp_mime_types":["application\/x-shockwave-flash"],"trans_depth":1,"protocol":"http","original_string":"HTTP | id.orig_p:49185 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/ tags:[] uid:C7KeXZ1jvzj9qkSqt7 referrer:http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745 resp_mime_types:[\"application\
\\/x-shockwave-flash\"] trans_depth:1 host:ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in status_msg:OK id.orig_h:192.168.138.158 response_body_len:8973 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978704.958145 id.resp_h:62.75.195.236 resp_fuids:[\"Ft6fqj1vE6fmJBYPx6\"]","ip_dst_addr":"62.75.195.236","threatinteljoinbolt.joiner.ts":"1530978710501","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","enrichmentjoinbolt.joiner.ts":"1530978710487","adapter.hostfromjsonlistadapter.begin.ts":"1530978710475","threatintelsplitterbolt.splitter.begin.ts":"1530978710493","enrichments.geo.ip_dst_addr.longitude":"7.7455","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30
729; Media Center PC 6.0)","resp_fuids":["Ft6fqj1vE6fmJBYPx6"],"timestamp":1530978704958,"method":"GET","request_body_len":0,"uri":"\/","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710476","referrer":"http:\/\/va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in\/?285a4d4e4e5a4d4d4649584c5d43064b4745","threatintelsplitterbolt.splitter.end.ts":"1530978710493","adapter.threatinteladapter.begin.ts":"1530978710497","ip_src_port":49185,"enrichments.geo.ip_dst_addr.location_point":"48.5839,7.7455","status_msg":"OK","guid":"f1d5ef09-d2e5-4cdd-a26b-fc23df82c385","response_body_len":8973}
+{"adapter.threatinteladapter.end.ts":"1530978710497","bro_timestamp":"1530978704.608287","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710477","enrichmentsplitterbolt.splitter.begin.ts":"1530978710477","adapter.hostfromjsonlistadapter.end.ts":"1530978710486","adapter.geoadapter.begin.ts":"1530978710486","uid":"CUrRne3iLIxXavQtci","trans_depth":40,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/persist\/wizard-data?_=1484168498643 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:40 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.608287 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710501","host":"node1","enrichmentjoinbolt.joiner.ts":"1530978710491","adapter.hostf
romjsonlistadapter.begin.ts":"1530978710486","threatintelsplitterbolt.splitter.begin.ts":"1530978710493","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704608,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/persist\/wizard-data?_=1484168498643","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710486","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710493","adapter.threatinteladapter.begin.ts":"1530978710497","ip_src_port":50451,"guid":"438b5c9d-522a-4611-9d70-c3723645611b","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.063932","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710477","enrichmentsplitterbolt.splitter.begin.ts":"1530978710477","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","adapter.geoadapter.begin.ts":"1530978710490","uid":"CUrRne3iLIxXavQtci","trans_depth":178,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters\/metron_cluster\/components\/?ServiceComponentInfo\/component_name=APP_TIMELINE_SERVER|ServiceComponentInfo\/category=MASTER&fields=ServiceComponentInfo\/service_name,host_components\/HostRoles\/display_name,host_components\/HostRoles\/host_name,host_components\/HostRoles\/state,host_components\/HostRoles\/maintenance_state,host_components\/HostRoles\/stale_configs,host_components\/HostRoles\/ha_state,host_components\/HostRoles\/desired_admin_state,,host_components\/metrics\/jvm\/memHeapUsed
M,host_components\/metrics\/jvm\/HeapMemoryMax,host_components\/metrics\/jvm\/HeapMemoryUsed,host_components\/metrics\/jvm\/memHeapCommittedM,host_components\/metrics\/mapred\/jobtracker\/trackers_decommissioned,host_components\/metrics\/cpu\/cpu_wio,host_components\/metrics\/rpc\/client\/RpcQueueTime_avg_time,host_components\/metrics\/dfs\/FSNamesystem\/*,host_components\/metrics\/dfs\/namenode\/Version,host_components\/metrics\/dfs\/namenode\/LiveNodes,host_components\/metrics\/dfs\/namenode\/DeadNodes,host_components\/metrics\/dfs\/namenode\/DecomNodes,host_components\/metrics\/dfs\/namenode\/TotalFiles,host_components\/metrics\/dfs\/namenode\/UpgradeFinalized,host_components\/metrics\/dfs\/namenode\/Safemode,host_components\/metrics\/runtime\/StartTime,host_components\/metrics\/hbase\/master\/IsActiveMaster,host_components\/metrics\/hbase\/master\/MasterStartTime,host_components\/metrics\/hbase\/master\/MasterActiveTime,host_components\/metrics\/hbase\/master\/AverageLoad,host_c
omponents\/metrics\/master\/AssignmentManger\/ritCount,metrics\/api\/v1\/cluster\/summary,metrics\/api\/v1\/topology\/summary,metrics\/api\/v1\/nimbus\/summary,host_components\/metrics\/yarn\/Queue,host_components\/metrics\/yarn\/ClusterMetrics\/NumActiveNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumLostNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumUnhealthyNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumRebootedNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumDecommissionedNMs&minimal_response=true&_=1484169119448 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:178 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.063932 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710502","host":"node1","enrichmentjoinbolt.joiner.ts":"1530
978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704063,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster\/components\/?ServiceComponentInfo\/component_name=APP_TIMELINE_SERVER|ServiceComponentInfo\/category=MASTER&fields=ServiceComponentInfo\/service_name,host_components\/HostRoles\/display_name,host_components\/HostRoles\/host_name,host_components\/HostRoles\/state,host_components\/HostRoles\/maintenance_state,host_components\/HostRoles\/stale_configs,host_components\/HostRoles\/ha_state,host_components\/HostRoles\/desired_admin_state,,host_components\/metrics\/jvm\/memHeapUsedM,host_components\/metrics\/jvm\/HeapMemoryMax,host_components\/metrics\/jvm\/HeapMemoryUsed,host_components\/met
rics\/jvm\/memHeapCommittedM,host_components\/metrics\/mapred\/jobtracker\/trackers_decommissioned,host_components\/metrics\/cpu\/cpu_wio,host_components\/metrics\/rpc\/client\/RpcQueueTime_avg_time,host_components\/metrics\/dfs\/FSNamesystem\/*,host_components\/metrics\/dfs\/namenode\/Version,host_components\/metrics\/dfs\/namenode\/LiveNodes,host_components\/metrics\/dfs\/namenode\/DeadNodes,host_components\/metrics\/dfs\/namenode\/DecomNodes,host_components\/metrics\/dfs\/namenode\/TotalFiles,host_components\/metrics\/dfs\/namenode\/UpgradeFinalized,host_components\/metrics\/dfs\/namenode\/Safemode,host_components\/metrics\/runtime\/StartTime,host_components\/metrics\/hbase\/master\/IsActiveMaster,host_components\/metrics\/hbase\/master\/MasterStartTime,host_components\/metrics\/hbase\/master\/MasterActiveTime,host_components\/metrics\/hbase\/master\/AverageLoad,host_components\/metrics\/master\/AssignmentManger\/ritCount,metrics\/api\/v1\/cluster\/summary,metrics\/api\/v1\/topol
ogy\/summary,metrics\/api\/v1\/nimbus\/summary,host_components\/metrics\/yarn\/Queue,host_components\/metrics\/yarn\/ClusterMetrics\/NumActiveNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumLostNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumUnhealthyNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumRebootedNMs,host_components\/metrics\/yarn\/ClusterMetrics\/NumDecommissionedNMs&minimal_response=true&_=1484169119448","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":50451,"guid":"03546910-68c3-4aa4-90a3-6983bc23324e","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.137918","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978710478","enrichments.geo.ip_dst_addr.city":"Elektrostal","enrichments.geo.ip_dst_addr.latitude":"55.7896","enrichmentsplitterbolt.splitter.begin.ts":"1530978710478","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","enrichments.geo.ip_dst_addr.country":"RU","enrichments.geo.ip_dst_addr.locID":"563523","adapter.geoadapter.begin.ts":"1530978710490","enrichments.geo.ip_dst_addr.postalCode":"144004","uid":"Cx8Ucg1r67RywyWab1","resp_mime_types":["image\/png"],"trans_depth":2,"protocol":"http","original_string":"HTTP | id.orig_p:49205 status_code:200 method:GET request_body_len:0 id.resp_p:80 uri:\/img\/flags\/us.png tags:[] uid:Cx8Ucg1r67RywyWab1 referrer:http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg resp_mime_types:[\"image\\\/png\"] trans_depth:2 host:7oqnsnzwwnm6zb7y.gigapaysun.com status_msg:OK id.orig_
h:192.168.138.158 response_body_len:825 user_agent:Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0) ts:1530978704.137918 id.resp_h:95.163.121.204 resp_fuids:[\"FCr63p4t8M7SUAumi3\"]","ip_dst_addr":"95.163.121.204","threatinteljoinbolt.joiner.ts":"1530978710502","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","enrichmentjoinbolt.joiner.ts":"1530978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","enrichments.geo.ip_dst_addr.longitude":"38.4467","ip_src_addr":"192.168.138.158","user_agent":"Mozilla\/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident\/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","resp_fuids":["FCr63p4t8M7SUAumi3"],"timestamp":1530978704137,"method":"GET","request_body_len":0,"uri":"\/img\/flags\/us.png","tags":[],"source.type":"bro","
adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/7oqnsnzwwnm6zb7y.gigapaysun.com\/11iQmfg","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":49205,"enrichments.geo.ip_dst_addr.location_point":"55.7896,38.4467","status_msg":"OK","guid":"5ea15274-bf38-423e-9c3e-6fb0f3bf0270","response_body_len":825}
+{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.973595","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710478","enrichmentsplitterbolt.splitter.begin.ts":"1530978710478","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","adapter.geoadapter.begin.ts":"1530978710490","uid":"CUrRne3iLIxXavQtci","trans_depth":251,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters\/metron_cluster?fields=Clusters\/desired_configs\/cluster-env&_=1484169429016 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:251 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.973595 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710502","host":"node1","enrichme
ntjoinbolt.joiner.ts":"1530978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704973,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster?fields=Clusters\/desired_configs\/cluster-env&_=1484169429016","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":50451,"guid":"273e3d59-b616-424e-8c30-add81bd671b9","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.973117","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710478","enrichmentsplitterbolt.splitter.begin.ts":"1530978710478","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","adapter.geoadapter.begin.ts":"1530978710490","uid":"CUrRne3iLIxXavQtci","trans_depth":247,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters?fields=Clusters\/provisioning_state&_=1484169420015 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/ trans_depth:247 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.973117 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710502","host":"node1","enrichmentjoinbolt.joiner.ts":"153
0978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704973,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters?fields=Clusters\/provisioning_state&_=1484169420015","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":50451,"guid":"9ba63d37-9e6b-4ba8-8504-cc418d9ce8aa","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978710499","qclass_name":"C_INTERNET","bro_timestamp":"1530978704.094553","qtype_name":"PTR","ip_dst_port":5353,"enrichmentsplitterbolt.splitter.end.ts":"1530978710479","qtype":12,"rejected":false,"enrichmentsplitterbolt.splitter.begin.ts":"1530978710479","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","trans_id":0,"adapter.geoadapter.begin.ts":"1530978710490","uid":"C03Lir2lgO0AxyDctk","protocol":"dns","original_string":"DNS | AA:false qclass_name:C_INTERNET id.orig_p:5353 qtype_name:PTR qtype:12 rejected:false id.resp_p:5353 query:_googlecast._tcp.local trans_id:0 TC:false RA:false uid:C03Lir2lgO0AxyDctk RD:false proto:udp id.orig_h:192.168.66.1 Z:0 qclass:1 ts:1530978704.094553 id.resp_h:224.0.0.251","ip_dst_addr":"224.0.0.251","threatinteljoinbolt.joiner.ts":"1530978710502","enrichmentjoinbolt.joiner.ts":"1530978710494","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"153097
8710497","Z":0,"ip_src_addr":"192.168.66.1","qclass":1,"timestamp":1530978704094,"AA":false,"query":"_googlecast._tcp.local","TC":false,"RA":false,"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","RD":false,"threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":5353,"proto":"udp","guid":"b5849fa0-3b1f-44a8-8b89-0eb3e823ba6f"}
+{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.896579","ip_dst_port":8080,"enrichmentsplitterbolt.splitter.end.ts":"1530978710479","enrichmentsplitterbolt.splitter.begin.ts":"1530978710479","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","adapter.geoadapter.begin.ts":"1530978710490","uid":"CUrRne3iLIxXavQtci","trans_depth":132,"protocol":"http","original_string":"HTTP | id.orig_p:50451 method:GET request_body_len:0 id.resp_p:8080 uri:\/api\/v1\/clusters\/metron_cluster\/components\/?fields=ServiceComponentInfo\/service_name,ServiceComponentInfo\/category,ServiceComponentInfo\/installed_count,ServiceComponentInfo\/started_count,ServiceComponentInfo\/init_count,ServiceComponentInfo\/install_failed_count,ServiceComponentInfo\/unknown_count,ServiceComponentInfo\/total_count,ServiceComponentInfo\/display_name,host_components\/HostRoles\/host_name&minimal_response=true&_=1484168884281 tags:[] uid:CUrRne3iLIxXavQtci referrer:http:\/\/node1:8080\/
trans_depth:132 host:node1 id.orig_h:192.168.66.1 response_body_len:0 user_agent:Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36 ts:1530978704.896579 id.resp_h:192.168.66.121","ip_dst_addr":"192.168.66.121","threatinteljoinbolt.joiner.ts":"1530978710502","host":"node1","enrichmentjoinbolt.joiner.ts":"1530978710495","adapter.hostfromjsonlistadapter.begin.ts":"1530978710489","threatintelsplitterbolt.splitter.begin.ts":"1530978710497","ip_src_addr":"192.168.66.1","user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/55.0.2883.95 Safari\/537.36","timestamp":1530978704896,"method":"GET","request_body_len":0,"uri":"\/api\/v1\/clusters\/metron_cluster\/components\/?fields=ServiceComponentInfo\/service_name,ServiceComponentInfo\/category,ServiceComponentInfo\/installed_count,ServiceComponentInfo\/started_count,ServiceComponentInfo\/init_count,ServiceComponent
Info\/install_failed_count,ServiceComponentInfo\/unknown_count,ServiceComponentInfo\/total_count,ServiceComponentInfo\/display_name,host_components\/HostRoles\/host_name&minimal_response=true&_=1484168884281","tags":[],"source.type":"bro","adapter.geoadapter.end.ts":"1530978710490","referrer":"http:\/\/node1:8080\/","threatintelsplitterbolt.splitter.end.ts":"1530978710497","adapter.threatinteladapter.begin.ts":"1530978710499","ip_src_port":50451,"guid":"bc257399-d461-4dd2-b6ac-d18c26af2dd2","response_body_len":0}
+{"adapter.threatinteladapter.end.ts":"1530978710499","bro_timestamp":"1530978704.015832","status_code":200,"ip_dst_port":80,"enrichmentsplitterbolt.splitter.end.ts":"1530978710479","enrichments.geo.ip_dst_addr.city":"Los Angeles","enrichments.geo.ip_dst_addr.latitude":"34.0494","enrichmentsplitterbolt.splitter.begin.ts":"1530978710479","adapter.hostfromjsonlistadapter.end.ts":"1530978710489","enrichments.geo.ip_dst_addr.country":"US","enrichments.geo.ip_dst_addr.locID":"5368361","adapter.geoadapter.begin.ts":"1530978710490","enrichments.geo.ip_dst_addr.postalCode":"90014","uid":"CpBTZB1XlDvW4TC9o4","resp_mime_types":["image\/png"],"trans
<TRUNCATED>
[4/4] metron git commit: METRON-1707 Port Profiler to Spark
(nickwallen) closes apache/metron#1150
Posted by ni...@apache.org.
METRON-1707 Port Profiler to Spark (nickwallen) closes apache/metron#1150
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3bfbf018
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3bfbf018
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3bfbf018
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 3bfbf018a9c3e1c74dc934901446b5111a0ada03
Parents: 6fb50a1
Author: nickwallen <ni...@nickallen.org>
Authored: Thu Aug 23 17:58:18 2018 -0400
Committer: nickallen <ni...@apache.org>
Committed: Thu Aug 23 17:58:18 2018 -0400
----------------------------------------------------------------------
dependencies_with_url.csv | 64 +++++-
.../profiler/DefaultMessageDistributor.java | 1 -
.../apache/metron/profiler/MessageRoute.java | 10 +-
metron-analytics/metron-profiler-spark/pom.xml | 195 +++++++++++++++++++
.../metron/profiler/spark/BatchProfiler.java | 102 ++++++++++
.../profiler/spark/BatchProfilerConfig.java | 190 ++++++++++++++++++
.../spark/ProfileMeasurementAdapter.java | 132 +++++++++++++
.../spark/function/GroupByPeriodFunction.java | 60 ++++++
.../spark/function/HBaseWriterFunction.java | 171 ++++++++++++++++
.../spark/function/MessageRouterFunction.java | 113 +++++++++++
.../spark/function/ProfileBuilderFunction.java | 107 ++++++++++
.../profiler/spark/function/TaskUtils.java | 41 ++++
.../spark/BatchProfilerIntegrationTest.java | 111 +++++++++++
.../spark/function/HBaseWriterFunctionTest.java | 176 +++++++++++++++++
.../function/MessageRouterFunctionTest.java | 114 +++++++++++
.../function/ProfileBuilderFunctionTest.java | 98 ++++++++++
.../src/test/resources/log4j.properties | 31 +++
.../src/test/resources/telemetry.json | 100 ++++++++++
metron-analytics/pom.xml | 1 +
.../configuration/profiler/ProfileResult.java | 4 +
.../profiler/ProfileResultExpressions.java | 4 +
pom.xml | 1 +
22 files changed, 1822 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 6ac1f23..6b4385b 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -33,13 +33,18 @@ com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache v2,https://github.com/maxmind
com.sun.xml.bind:jaxb-impl:jar:2.2.3-1:compile,CDDL,http://jaxb.java.net/
com.sun.xml.bind:jaxb-impl:jar:2.2.5-2:compile,CDDL,http://jaxb.java.net/
com.twitter:jsr166e:jar:1.1.0:compile,CC0 1.0 Universal,http://github.com/twitter/jsr166e
+com.twitter:chill-java:jar:0.8.4:compile,ASLv2,https://github.com/twitter/chill
+com.twitter:chill_2.11:jar:0.8.4:compile,ASLv2,https://github.com/twitter/chill
it.unimi.dsi:fastutil:jar:7.0.6:compile,ASLv2,https://github.com/vigna/fastutil
javassist:javassist:jar:3.12.1.GA:compile,Apache v2,http://www.javassist.org/
javax.activation:activation:jar:1.1:compile,Common Development and Distribution License (CDDL) v1.0,http://java.sun.com/products/javabeans/jaf/index.jsp
+javax.activation:activation:jar:1.1.1:compile,Common Development and Distribution License (CDDL) v1.0,http://java.sun.com/products/javabeans/jaf/index.jsp
javax.annotation:jsr250-api:jar:1.0:compile,COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0,http://jcp.org/aboutJava/communityprocess/final/jsr250/index.html
javax.annotation:javax.annotation-api:jar:1.3.2:compile,CDDL 1.1,https://github.com/javaee/javax.annotation/
+javax.annotation:javax.annotation-api:jar:1.2:compile,CDDL 1.1,https://github.com/javaee/javax.annotation/
javax.mail:mail:jar:1.4:compile,Common Development and Distribution License (CDDL) v1.0,https://glassfish.dev.java.net/javaee5/mail/
javax.servlet:javax.servlet-api:jar:3.1.0:compile,CDDL,http://servlet-spec.java.net
+javax.ws.rs:javax.ws.rs-api:jar:2.0.1:compile,CDDL 1.1,https://github.com/jax-rs/api
javax.xml.bind:jaxb-api:jar:2.2.11:compile,CDDL,http://jaxb.java.net/
javax.xml.bind:jaxb-api:jar:2.2.2:compile,CDDL,https://jaxb.dev.java.net/
javax.xml.bind:jaxb-api:jar:2.3.0:compile,CDDL,https://jaxb.dev.java.net/
@@ -47,25 +52,41 @@ javax.xml.stream:stax-api:jar:1.0-2:compile,COMMON DEVELOPMENT AND DISTRIBUTION
jline:jline:jar:0.9.94:compile,BSD,http://jline.sourceforge.net
junit:junit:jar:4.12:compile,Eclipse Public License 1.0,http://junit.org
junit:junit:jar:4.4:compile,Common Public License Version 1.0,http://junit.org
+net.razorvine:pyrolite:jar:4.13:compile,MIT,https://github.com/irmen/Pyrolite
net.sf.jopt-simple:jopt-simple:jar:3.2:compile,The MIT License,http://jopt-simple.sourceforge.net
net.sf.jopt-simple:jopt-simple:jar:4.9:compile,The MIT License,http://jopt-simple.sourceforge.net
net.sf.saxon:Saxon-HE:jar:9.5.1-5:compile,Mozilla Public License Version 2.0,http://www.saxonica.com/
org.abego.treelayout:org.abego.treelayout.core:jar:1.0.1:compile,BSD 3-Clause "New" or "Revised" License (BSD-3-Clause),http://code.google.com/p/treelayout/
org.adrianwalker:multiline-string:jar:0.1.2:compile,Common Public License Version 1.0,https://github.com/benelog/multiline
org.antlr:antlr4-runtime:jar:4.5:compile,BSD 3-Clause License,http://www.antlr.org
+org.bouncycastle:bcprov-jdk15on:jar:1.52:compile,MIT,https://www.bouncycastle.org/license.html
org.clojure:clojure:jar:1.6.0:compile,Eclipse Public License 1.0,http://clojure.org/
org.clojure:clojure:jar:1.7.0:compile,Eclipse Public License 1.0,http://clojure.org/
org.codehaus.jackson:jackson-jaxrs:jar:1.8.3:compile,Apache v2,http://jackson.codehaus.org
org.codehaus.jackson:jackson-jaxrs:jar:1.9.13:compile,Apache v2,http://jackson.codehaus.org
org.codehaus.jackson:jackson-xc:jar:1.8.3:compile,Apache v2,http://jackson.codehaus.org
org.codehaus.jackson:jackson-xc:jar:1.9.13:compile,Apache v2,http://jackson.codehaus.org
+org.codehaus.janino:commons-compiler:jar:3.0.8:compile,New BSD,https://github.com/janino-compiler/janino
+org.codehaus.janino:janino:jar:3.0.8:compile,New BSD,https://github.com/janino-compiler/janino
org.codehaus.woodstox:stax2-api:jar:3.1.4:compile,The BSD License,http://wiki.fasterxml.com/WoodstoxStax2
+org.json4s:json4s-ast_2.11:jar:3.2.11:compile,ASLv2,https://github.com/json4s/json4s
+org.json4s:json4s-core_2.11:jar:3.2.11:compile,ASLv2,https://github.com/json4s/json4s
+org.json4s:json4s-jackson_2.11:jar:3.2.11:compile,ASLv2,https://github.com/json4s/json4s
org.jruby.jcodings:jcodings:jar:1.0.8:compile,MIT License,https://github.com/jruby/jcodings
org.jruby.joni:joni:jar:2.1.2:compile,MIT License,https://github.com/jruby/joni
+org.lz4:lz4-java:jar:1.4.0:compile,ASLv2,https://github.com/lz4/lz4-java
org.mitre.taxii:taxii:jar:1.1.0.1:compile,The BSD 3-Clause License,https://github.com/TAXIIProject/java-taxii
org.mitre:stix:jar:1.2.0.2:compile,The BSD 3-Clause License,https://github.com/STIXProject/java-stix
org.mockito:mockito-core:jar:1.10.19:compile,The MIT License,http://www.mockito.org
+org.roaringbitmap:RoaringBitmap:jar:0.5.11:compile,ASLv2,https://github.com/RoaringBitmap/RoaringBitmap
org.scala-lang:scala-library:jar:2.10.6:compile,BSD-like,http://www.scala-lang.org/
+org.scala-lang.modules:scala-parser-combinators_2.11:jar:1.0.4:compile,BSD-like,http://www.scala-lang.org/
+org.scala-lang.modules:scala-xml_2.11:jar:1.0.1:compile,BSD-like,http://www.scala-lang.org/
+org.scala-lang:scala-compiler:jar:2.11.0:compile,BSD-like,http://www.scala-lang.org/
+org.scala-lang:scala-library:jar:2.11.8:compile,BSD-like,http://www.scala-lang.org/
+org.scala-lang:scala-reflect:jar:2.11.8:compile,BSD-like,http://www.scala-lang.org/
+org.scala-lang:scalap:jar:2.11.0:compile,BSD-like,http://www.scala-lang.org/
+oro:oro:jar:2.0.8:compile,ASLv2,http://attic.apache.org/projects/jakarta-oro.html
xmlenc:xmlenc:jar:0.52:compile,The BSD License,http://xmlenc.sourceforge.net
asm:asm:jar:3.1:compile,BSD,http://asm.ow2.org/
com.sun.jersey.contribs:jersey-guice:jar:1.9:compile,CDDL 1.1,https://jersey.java.net/
@@ -103,7 +124,10 @@ org.slf4j:slf4j-log4j12:jar:1.7.5:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-log4j12:jar:1.7.7:compile,MIT,http://www.slf4j.org
org.slf4j:slf4j-simple:jar:1.7.7:compile,MIT,http://www.slf4j.org
org.slf4j:jcl-over-slf4j:jar:1.7.7:compile,MIT,http://www.slf4j.org
+org.slf4j:jcl-over-slf4j:jar:1.7.16:compile,MIT,http://www.slf4j.org
org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
+org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
+org.slf4j:jul-to-slf4j:jar:1.7.16:compile,MIT,http://www.slf4j.org
org.slf4j:jul-to-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org
org.slf4j:jul-to-slf4j:jar:1.7.25:compile,MIT,http://www.slf4j.org
aopalliance:aopalliance:jar:1.0:compile,Public Domain,http://aopalliance.sourceforge.net
@@ -113,7 +137,9 @@ com.github.tony19:named-regexp:jar:0.2.3:compile,Apache License, Version 2.0,
com.google.code.findbugs:jsr305:jar:1.3.9:compile,The Apache Software License, Version 2.0,http://findbugs.sourceforge.net/
com.google.code.findbugs:jsr305:jar:3.0.0:compile,The Apache Software License, Version 2.0,http://findbugs.sourceforge.net/
com.google.code.findbugs:annotations:jar:2.0.1:compile,The Apache Software License, Version 2.0,http://findbugs.sourceforge.net/
-com.carrotsearch:hppc:jar:0.7.1:compile,ASLv2,
+com.carrotsearch:hppc:jar:0.7.1:compile,ASLv2,https://github.com/carrotsearch/hppc
+com.carrotsearch:hppc:jar:0.7.2:compile,ASLv2,https://github.com/carrotsearch/hppc
+com.clearspring.analytics:stream:jar:2.7.0:compile,ASLv2,https://github.com/addthis/stream-lib
com.clearspring.analytics:stream:jar:2.9.5:compile,ASLv2,https://github.com/addthis/stream-lib
com.codahale.metrics:metrics-core:jar:3.0.2:compile,MIT,https://github.com/codahale/metrics
com.codahale.metrics:metrics-graphite:jar:3.0.2:compile,MIT,https://github.com/codahale/metrics
@@ -132,6 +158,7 @@ com.fasterxml.jackson.core:jackson-core:jar:2.9.4:compile,ASLv2,https://github.c
com.fasterxml.jackson.core:jackson-core:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-core
com.fasterxml.jackson.core:jackson-databind:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome
com.fasterxml.jackson.core:jackson-databind:jar:2.4.3:compile,ASLv2,http://github.com/FasterXML/jackson
+com.fasterxml.jackson.core:jackson-databind:jar:2.6.7.1:compile,ASLv2,http://github.com/FasterXML/jackson
com.fasterxml.jackson.core:jackson-databind:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson
com.fasterxml.jackson.core:jackson-databind:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson
com.fasterxml.jackson.core:jackson-databind:jar:2.9.4:compile,ASLv2,http://github.com/FasterXML/jackson
@@ -147,6 +174,8 @@ com.fasterxml.jackson.datatype:jackson-datatype-joda:jar:2.9.5:compile,ASLv2,htt
com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8
com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8
com.fasterxml.jackson.module:jackson-module-parameter-names:jar:2.9.5:compile,ASLv2,https://github.com/FasterXML/jackson-modules-java8
+com.fasterxml.jackson.module:jackson-module-paranamer:jar:2.7.9:compile,ASLv2,https://github.com/FasterXML/jackson-modules-base
+com.fasterxml.jackson.module:jackson-module-scala_2.11:jar:2.6.7.1:compile,ASLv2,https://github.com/FasterXML/jackson-module-scala
com.fasterxml:classmate:jar:1.3.1:compile,ASLv2,http://github.com/cowtowncoder/java-classmate
com.fasterxml:classmate:jar:1.3.4:compile,ASLv2,http://github.com/cowtowncoder/java-classmate
com.google.code.gson:gson:jar:2.2.4:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/
@@ -164,10 +193,14 @@ com.lmax:disruptor:jar:3.3.2:compile,The Apache Software License, Version 2.0,ht
com.googlecode.json-simple:json-simple:jar:1.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/json-simple/
com.googlecode.json-simple:json-simple:jar:1.1.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/json-simple/
com.jamesmurty.utils:java-xmlbuilder:jar:0.4:compile,Apache License, Version 2.0,http://code.google.com/p/java-xmlbuilder/
+com.jamesmurty.utils:java-xmlbuilder:jar:1.1:compile,Apache License, Version 2.0,http://code.google.com/p/java-xmlbuilder/
com.ning:compress-lzf:jar:1.0.2:compile,Apache License 2.0,http://github.com/ning/compress
+com.ning:compress-lzf:jar:1.0.3:compile,Apache License 2.0,http://github.com/ning/compress
com.opencsv:opencsv:jar:3.7:compile,Apache 2,http://opencsv.sf.net
com.spatial4j:spatial4j:jar:0.5:compile,The Apache Software License, Version 2.0,
com.tdunning:t-digest:jar:3.0:compile,The Apache Software License, Version 2.0,https://github.com/tdunning/t-digest
+com.univocity:univocity-parsers:jar:2.5.9:compile,ASLv2,https://github.com/uniVocity/univocity-parsers
+com.vlkan:flatbuffers:jar:1.2.0-3f79e055:compile,ASLv2,https://github.com/vy/flatbuffers
com.yammer.metrics:metrics-core:jar:2.2.0:compile,ASLv2,
commons-beanutils:commons-beanutils-core:jar:1.8.0:compile,ASLv2,http://commons.apache.org/beanutils/
commons-beanutils:commons-beanutils-core:jar:1.8.0:provided,ASLv2,http://commons.apache.org/beanutils/
@@ -203,19 +236,28 @@ commons-lang:commons-lang:jar:2.6:provided,ASLv2,http://commons.apache.org/lang/
commons-logging:commons-logging:jar:1.1.1:compile,ASLv2,http://commons.apache.org/logging
commons-logging:commons-logging:jar:1.1.3:compile,ASLv2,http://commons.apache.org/proper/commons-logging/
commons-logging:commons-logging:jar:1.2:compile,ASLv2,http://commons.apache.org/proper/commons-logging/
+commons-net:commons-net:jar:2.2:compile,ASLv2,http://commons.apache.org/net/
commons-net:commons-net:jar:3.1:compile,ASLv2,http://commons.apache.org/net/
commons-net:commons-net:jar:3.1:provided,ASLv2,http://commons.apache.org/net/
commons-text:commons-text:jar:1.1:compile,ASLv2,http://commons.apache.org/proper/commons-text/
commons-validator:commons-validator:jar:1.4.0:compile,ASLv2,http://commons.apache.org/validator/
commons-validator:commons-validator:jar:1.5.1:compile,ASLv2,http://commons.apache.org/proper/commons-validator/
commons-validator:commons-validator:jar:1.6:compile,ASLv2,http://commons.apache.org/proper/commons-validator/
+et.razorvine:pyrolite:jar:4.13:compile,MIT,https://github.com/irmen/Pyrolite
+io.airlift:aircompressor:jar:0.8:compile,ASLv2,https://github.com/airlift/aircompressor
io.confluent:kafka-avro-serializer:jar:1.0:compile,ASLv2,https://github.com/confluentinc/schema-registry/
io.confluent:kafka-schema-registry-client:jar:1.0:compile,ASLv2,https://github.com/confluentinc/schema-registry/
+io.dropwizard.metrics:metrics-core:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics
+io.dropwizard.metrics:metrics-graphite:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics
+io.dropwizard.metrics:metrics-json:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics
+io.dropwizard.metrics:metrics-jvm:jar:3.1.5:compile,ASLv2,https://github.com/dropwizard/metrics
io.netty:netty-all:jar:4.0.23.Final:compile,ASLv2,
io.netty:netty-all:jar:4.0.23.Final:provided,ASLv2,
-io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/
+io.netty:netty-all:jar:4.1.17.Final:compile,ASLv2,
io.netty:netty:jar:3.6.2.Final:compile,Apache License, Version 2.0,http://netty.io/
io.netty:netty:jar:3.7.0.Final:compile,Apache License, Version 2.0,http://netty.io/
+io.netty:netty:jar:3.9.9.Final:compile,Apache License, Version 2.0,http://netty.io/
+io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/
io.thekraken:grok:jar:0.1.0:compile,Apache License, Version 2.0,http://maven.apache.org
javax.inject:javax.inject:jar:1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/atinject/
joda-time:joda-time:jar:2.3:compile,Apache 2,http://www.joda.org/joda-time/
@@ -224,9 +266,12 @@ joda-time:joda-time:jar:2.9.9:compile,Apache 2,http://www.joda.org/joda-time/
log4j:log4j:jar:1.2.15:compile,The Apache Software License, Version 2.0,http://logging.apache.org:80/log4j/1.2/
log4j:log4j:jar:1.2.16:compile,The Apache Software License, Version 2.0,http://logging.apache.org/log4j/1.2/
log4j:log4j:jar:1.2.17:compile,The Apache Software License, Version 2.0,http://logging.apache.org/log4j/1.2/
+net.iharder:base64:jar:2.3.8:compile,Public Domain,http://iharder.sourceforge.net/current/java/base64/
net.java.dev.jets3t:jets3t:jar:0.9.0:compile,Apache License, Version 2.0,http://www.jets3t.org
+net.java.dev.jets3t:jets3t:jar:0.9.4:compile,Apache License, Version 2.0,http://www.jets3t.org
net.jpountz.lz4:lz4:jar:1.2.0:compile,The Apache Software License, Version 2.0,https://github.com/jpountz/lz4-java
net.jpountz.lz4:lz4:jar:1.3.0:compile,The Apache Software License, Version 2.0,https://github.com/jpountz/lz4-java
+net.sf.py4j:py4j:jar:0.10.7:compile,,
nl.jqno.equalsverifier:equalsverifier:jar:2.0.2:compile,The Apache Software License, Version 2.0,http://www.jqno.nl/equalsverifier
org.codehaus.jackson:jackson-core-asl:jar:1.9.13:compile,The Apache Software License, Version 2.0,http://jackson.codehaus.org
org.codehaus.jackson:jackson-mapper-asl:jar:1.9.13:compile,The Apache Software License, Version 2.0,http://jackson.codehaus.org
@@ -354,6 +399,7 @@ org.springframework.security:spring-security-core:jar:4.1.3.RELEASE:compile,ASLv
org.springframework.security:spring-security-core:jar:5.0.4.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
org.springframework.security:spring-security-web:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
org.springframework.security:spring-security-web:jar:5.0.4.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security
+org.spark-project.spark:unused:jar:1.0.0:compile,ASLv2,https://spark.apache.org
antlr:antlr:jar:2.7.7:compile,BSD 3-Clause License,http://www.antlr2.org
com.h2database:h2:jar:1.4.192:compile,EPL 1.0,http://www.h2database.com/html/license.html
com.h2database:h2:jar:1.4.197:compile,EPL 1.0,http://www.h2database.com/html/license.html
@@ -370,6 +416,7 @@ org.springframework.kafka:spring-kafka:jar:1.1.1.RELEASE:compile,ASLv2,https://g
org.springframework.kafka:spring-kafka:jar:2.0.4.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-kafka
ch.hsr:geohash:jar:1.3.0:compile,ASLv2,https://github.com/kungfoo/geohash-java
org.locationtech.spatial4j:spatial4j:jar:0.6:compile,ASLv2,https://github.com/locationtech/spatial4j
+com.github.luben:zstd-jni:jar:1.3.2-2:compile,BSD,https://github.com/luben/zstd-jni
com.github.spullara.mustache.java:compiler:jar:0.9.3:compile,ASLv2,https://github.com/spullara/mustache.java/blob/master/LICENSE
io.netty:netty-buffer:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
io.netty:netty-codec-http:jar:4.1.13.Final:compile,ASLv2,http://netty.io/
@@ -395,6 +442,19 @@ org.elasticsearch:securesm:jar:1.1:compile,ASLv2,https://github.com/elastic/elas
org.hdrhistogram:HdrHistogram:jar:2.1.9:compile,BSD,https://github.com/HdrHistogram/HdrHistogram/blob/master/LICENSE.txt
com.trendmicro:tlsh:jar:3.7.1:compile,ASLv2,https://github.com/trendmicro/tlsh
org.glassfish:javax.json:jar:1.0.4:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/jsonp
+org.glassfish.hk2.external:aopalliance-repackaged:jar:2.4.0-b34:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/hk2
+org.glassfish.hk2.external:javax.inject:jar:2.4.0-b34:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/hk2
+org.glassfish.hk2:hk2-api:jar:2.4.0-b34:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/hk2
+org.glassfish.hk2:hk2-locator:jar:2.4.0-b34:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/hk2
+org.glassfish.hk2:hk2-utils:jar:2.4.0-b34:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/hk2
+org.glassfish.hk2:osgi-resource-locator:jar:1.0.1:compile,Common Development and Distribution License (CDDL) v1.0,https://github.com/javaee/hk2
+org.glassfish.jersey.bundles.repackaged:jersey-guava:jar:2.22.2:compile
+org.glassfish.jersey.containers:jersey-container-servlet-core:jar:2.22.2:compile
+org.glassfish.jersey.containers:jersey-container-servlet:jar:2.22.2:compile
+org.glassfish.jersey.core:jersey-client:jar:2.22.2:compile,EPL 2.0,https://github.com/eclipse-ee4j/jersey
+org.glassfish.jersey.core:jersey-common:jar:2.22.2:compile,EPL 2.0,https://github.com/eclipse-ee4j/jersey
+org.glassfish.jersey.core:jersey-server:jar:2.22.2:compile,EPL 2.0,https://github.com/eclipse-ee4j/jersey
+org.glassfish.jersey.media:jersey-media-jaxb:jar:2.22.2:compile,EPL 2.0,https://github.com/eclipse-ee4j/jersey
org.eclipse.persistence:javax.persistence:jar:2.1.1:compile,EPL 1.0,http://www.eclipse.org/eclipselink
org.eclipse.persistence:org.eclipse.persistence.antlr:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink
org.eclipse.persistence:org.eclipse.persistence.asm:jar:2.6.4:compile,EPL 1.0,http://www.eclipse.org/eclipselink
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
index d950b07..673072b 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/DefaultMessageDistributor.java
@@ -28,7 +28,6 @@ import com.google.common.cache.RemovalNotification;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import org.apache.metron.stellar.dsl.Context;
-import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/MessageRoute.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/MessageRoute.java b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/MessageRoute.java
index e76b897..7cdb607 100644
--- a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/MessageRoute.java
+++ b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/MessageRoute.java
@@ -22,11 +22,11 @@ package org.apache.metron.profiler;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import org.json.simple.JSONObject;
import java.io.Serializable;
+import java.util.Map;
/**
* Defines the 'route' a message must take through the Profiler.
@@ -74,6 +74,10 @@ public class MessageRoute implements Serializable {
this.timestamp = timestamp;
}
+ public MessageRoute() {
+ // necessary for serialization
+ }
+
public String getEntity() {
return entity;
}
@@ -98,6 +102,10 @@ public class MessageRoute implements Serializable {
this.message = message;
}
+ public void setMessage(Map message) {
+ this.message = new JSONObject(message);
+ }
+
public Long getTimestamp() {
return timestamp;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/pom.xml b/metron-analytics/metron-profiler-spark/pom.xml
new file mode 100644
index 0000000..93ce08a
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/pom.xml
@@ -0,0 +1,195 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-analytics</artifactId>
+ <version>0.5.1</version>
+ </parent>
+ <artifactId>metron-profiler-spark</artifactId>
+ <url>https://metron.apache.org/</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.11</artifactId>
+ <version>${global_spark_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.11</artifactId>
+ <version>${global_spark_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.antlr</groupId>
+ <artifactId>antlr-runtime</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-profiler-common</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-common</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-hbase</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-hbase</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${global_hbase_version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <!-- allows profiles to use the Stellar stats functions -->
+ <groupId>org.apache.metron</groupId>
+ <artifactId>metron-statistics</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>kryo</artifactId>
+ <groupId>com.esotericsoftware</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ <version>${global_log4j_core_version}</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>${global_shade_version}</version>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.tdunning</pattern>
+ <shadedPattern>org.apache.metron.tdunning</shadedPattern>
+ </relocation>
+ </relocations>
+ <artifactSet>
+ <excludes>
+ <exclude>storm:storm-core:*</exclude>
+ <exclude>storm:storm-lib:*</exclude>
+ <exclude>org.slf4j.impl*</exclude>
+ <exclude>org.slf4j:slf4j-log4j*</exclude>
+ </excludes>
+ </artifactSet>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resources>
+ <resource>.yaml</resource>
+ <resource>LICENSE.txt</resource>
+ <resource>ASL2.0</resource>
+ <resource>NOTICE.txt</resource>
+ </resources>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass></mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
new file mode 100644
index 0000000..f999613
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfiler.java
@@ -0,0 +1,102 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark;
+
+import com.google.common.collect.Maps;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.profiler.MessageRoute;
+import org.apache.metron.profiler.spark.function.GroupByPeriodFunction;
+import org.apache.metron.profiler.spark.function.HBaseWriterFunction;
+import org.apache.metron.profiler.spark.function.MessageRouterFunction;
+import org.apache.metron.profiler.spark.function.ProfileBuilderFunction;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.lang.invoke.MethodHandles;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+import static org.apache.spark.sql.functions.sum;
+
+/**
+ * The 'Batch Profiler' that generates profiles by consuming data in batch from archived telemetry.
+ *
+ * <p>The Batch Profiler is executed in Spark.
+ */
+public class BatchProfiler implements Serializable {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Execute the Batch Profiler.
+ *
+ * @param spark The spark session.
+ * @param properties The profiler configuration properties.
+ * @param profiles The profile definitions.
+ * @return The number of profile measurements produced.
+ */
+ public long run(SparkSession spark,
+ Properties properties,
+ Properties globalProperties,
+ ProfilerConfig profiles) {
+
+ LOG.debug("Building {} profile(s)", profiles.getProfiles().size());
+ Map<String, String> globals = Maps.fromProperties(globalProperties);
+
+ String inputFormat = TELEMETRY_INPUT_FORMAT.get(properties, String.class);
+ String inputPath = TELEMETRY_INPUT_PATH.get(properties, String.class);
+ LOG.debug("Loading telemetry from '{}'", inputPath);
+
+ // fetch the archived telemetry
+ Dataset<String> telemetry = spark
+ .read()
+ .format(inputFormat)
+ .load(inputPath)
+ .as(Encoders.STRING());
+ LOG.debug("Found {} telemetry record(s)", telemetry.cache().count());
+
+ // find all routes for each message
+ Dataset<MessageRoute> routes = telemetry
+ .flatMap(new MessageRouterFunction(profiles, globals), Encoders.bean(MessageRoute.class));
+ LOG.debug("Generated {} message route(s)", routes.cache().count());
+
+ // build the profiles
+ Dataset<ProfileMeasurementAdapter> measurements = routes
+ .groupByKey(new GroupByPeriodFunction(properties), Encoders.STRING())
+ .mapGroups(new ProfileBuilderFunction(properties, globals), Encoders.bean(ProfileMeasurementAdapter.class));
+ LOG.debug("Produced {} profile measurement(s)", measurements.cache().count());
+
+ // write the profile measurements to HBase
+ long count = measurements
+ .mapPartitions(new HBaseWriterFunction(properties), Encoders.INT())
+ .agg(sum("value"))
+ .head()
+ .getLong(0);
+ LOG.debug("{} profile measurement(s) written to HBase", count);
+
+ return count;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
new file mode 100644
index 0000000..054806e
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/BatchProfilerConfig.java
@@ -0,0 +1,190 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark;
+
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Defines the configuration values recognized by the Batch Profiler.
+ */
+public enum BatchProfilerConfig {
+
+ PERIOD_DURATION_UNITS("profiler.period.duration.units", "MINUTES", String.class),
+
+ PERIOD_DURATION("profiler.period.duration", 15, Integer.class),
+
+ HBASE_SALT_DIVISOR("profiler.hbase.salt.divisor", 1000, Integer.class),
+
+ HBASE_TABLE_PROVIDER("profiler.hbase.table.provider", "org.apache.metron.hbase.HTableProvider", String.class),
+
+ HBASE_TABLE_NAME("profiler.hbase.table", "profiler", String.class),
+
+ HBASE_COLUMN_FAMILY("profiler.hbase.column.family", "P", String.class),
+
+ HBASE_WRITE_DURABILITY("profiler.hbase.durability", Durability.USE_DEFAULT, Durability.class),
+
+ TELEMETRY_INPUT_FORMAT("profiler.batch.input.format", "text", String.class),
+
+ TELEMETRY_INPUT_PATH("profiler.batch.input.path", "hdfs://localhost:9000/apps/metron/indexing/indexed/*/*", String.class);
+
+ /**
+ * The key for the configuration value.
+ */
+ private String key;
+
+ /**
+ * The default value of the configuration, if none other is specified.
+ */
+ private Object defaultValue;
+
+ /**
+ * The type of the configuration value.
+ */
+ private Class<?> valueType;
+
+ BatchProfilerConfig(String key, Object defaultValue, Class<?> valueType) {
+ this.key = key;
+ this.defaultValue = defaultValue;
+ this.valueType = valueType;
+ }
+
+ /**
+ * Returns the key of the configuration value.
+ */
+ public String getKey() {
+ return key;
+ }
+
+ /**
+ * Returns the default value of the configuration.
+ */
+ public Object getDefault() {
+ return getDefault(valueType);
+ }
+
+ /**
+ * Returns the default value of the configuration, cast to the expected type.
+ *
+ * @param clazz The class of the expected type of the configuration value.
+ * @param <T> The expected type of the configuration value.
+ */
+ public <T> T getDefault(Class<T> clazz) {
+ return defaultValue == null ? null: ConversionUtils.convert(defaultValue, clazz);
+ }
+
+ /**
+ * Returns the configuration value from a map of configuration values.
+ *
+ * @param config A map containing configuration values.
+ */
+ public Object get(Map<String, String> config) {
+ return getOrDefault(config, defaultValue);
+ }
+
+ /**
+ * Returns the configuration value from a map of configuration values.
+ *
+ * @param properties Configuration properties.
+ */
+ public Object get(Properties properties) {
+ return getOrDefault(properties, defaultValue);
+ }
+
+ /**
+ * Returns the configuration value from a map of configuration values, cast to the expected type.
+ *
+ * @param config A map containing configuration values.
+ */
+ public <T> T get(Map<String, String> config, Class<T> clazz) {
+ return getOrDefault(config, defaultValue, clazz);
+ }
+
+ /**
+ * Returns the configuration value from a map of configuration values, cast to the expected type.
+ *
+ * @param properties Configuration properties.
+ */
+ public <T> T get(Properties properties, Class<T> clazz) {
+ return getOrDefault(properties, defaultValue, clazz);
+ }
+
+ /**
+ * Returns the configuration value from a map of configuration values. If the value is not specified,
+ * the default value is returned.
+ *
+ * @param config A map containing configuration values.
+ * @param defaultValue The default value to return, if one is not defined.
+ * @return The configuration value or the specified default, if one is not defined.
+ */
+ private Object getOrDefault(Map<String, String> config, Object defaultValue) {
+ return getOrDefault(config, defaultValue, valueType);
+ }
+
+ /**
+ * Returns the configuration value from a map of configuration values. If the value is not specified,
+ * the default value is returned.
+ *
+ * @param properties A map containing configuration values.
+ * @param defaultValue The default value to return, if one is not defined.
+ * @return The configuration value or the specified default, if one is not defined.
+ */
+ private Object getOrDefault(Properties properties, Object defaultValue) {
+ return getOrDefault(properties, defaultValue, valueType);
+ }
+
+ /**
+ * Returns the configuration value, cast to the expected type, from a map of configuration values.
+ * If the value is not specified, the default value is returned.
+ *
+ * @param config A map containing configuration values.
+ * @param defaultValue The default value to return, if one is not defined.
+ * @param clazz The class of the expected type of the configuration value.
+ * @param <T> The expected type of the configuration value.
+ * @return The configuration value or the specified default, if one is not defined.
+ */
+ private <T> T getOrDefault(Map<String, String> config, Object defaultValue, Class<T> clazz) {
+ Object value = config.getOrDefault(key, defaultValue.toString());
+ return value == null ? null : ConversionUtils.convert(value, clazz);
+ }
+
+ /**
+ * Returns the configuration value, cast to the expected type, from a map of configuration values.
+ * If the value is not specified, the default value is returned.
+ *
+ * @param properties Configuration properties.
+ * @param defaultValue The default value to return, if one is not defined.
+ * @param clazz The class of the expected type of the configuration value.
+ * @param <T> The expected type of the configuration value.
+ * @return The configuration value or the specified default, if one is not defined.
+ */
+ private <T> T getOrDefault(Properties properties, Object defaultValue, Class<T> clazz) {
+ Object value = properties.getOrDefault(key, defaultValue);
+ return value == null ? null : ConversionUtils.convert(value, clazz);
+ }
+
+ @Override
+ public String toString() {
+ return key;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/ProfileMeasurementAdapter.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/ProfileMeasurementAdapter.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/ProfileMeasurementAdapter.java
new file mode 100644
index 0000000..5da7d04
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/ProfileMeasurementAdapter.java
@@ -0,0 +1,132 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark;
+
+import org.apache.metron.common.utils.SerDeUtils;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.ProfilePeriod;
+
+import java.io.Serializable;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * An adapter for the {@link ProfileMeasurement} class so that the data
+ * can be serialized as required by Spark.
+ *
+ * <p>The `Encoders.bean(Class<T>)` encoder does not handle serialization of type `Object` well. This
+ * adapter encodes the profile's result as byte[] rather than an Object to work around this.
+ */
+public class ProfileMeasurementAdapter implements Serializable {
+
+ /**
+ * The name of the profile that this measurement is associated with.
+ */
+ private String profileName;
+
+ /**
+ * The name of the entity being profiled.
+ */
+ private String entity;
+
+ /**
+ * A monotonically increasing number identifying the period. The first period is 0
+ * and began at the epoch.
+ */
+ private Long periodId;
+
+ /**
+ * The duration of each period in milliseconds.
+ */
+ private Long durationMillis;
+
+ /**
+ * The result of evaluating the profile expression.
+ *
+ * The `Encoders.bean(Class<T>)` encoder does not handle serialization of type `Object`. This
+ * adapter encodes the profile's result as `byte[]` rather than an `Object` to work around this.
+ */
+ private byte[] profileValue;
+
+ public ProfileMeasurementAdapter() {
+ // default constructor required for serialization in Spark
+ }
+
+ public ProfileMeasurementAdapter(ProfileMeasurement measurement) {
+ this.profileName = measurement.getProfileName();
+ this.entity = measurement.getEntity();
+ this.periodId = measurement.getPeriod().getPeriod();
+ this.durationMillis = measurement.getPeriod().getDurationMillis();
+ this.profileValue = SerDeUtils.toBytes(measurement.getProfileValue());
+ }
+
+ public ProfileMeasurement toProfileMeasurement() {
+ ProfilePeriod period = ProfilePeriod.fromPeriodId(periodId, durationMillis, TimeUnit.MILLISECONDS);
+ ProfileMeasurement measurement = new ProfileMeasurement()
+ .withProfileName(profileName)
+ .withEntity(entity)
+ .withPeriod(period)
+ .withProfileValue(SerDeUtils.fromBytes(profileValue, Object.class));
+ return measurement;
+ }
+
+ public String getProfileName() {
+ return profileName;
+ }
+
+ public void setProfileName(String profileName) {
+ this.profileName = profileName;
+ }
+
+ public String getEntity() {
+ return entity;
+ }
+
+ public void setEntity(String entity) {
+ this.entity = entity;
+ }
+
+ public Long getPeriodId() {
+ return periodId;
+ }
+
+ public void setPeriodId(Long periodId) {
+ this.periodId = periodId;
+ }
+
+ public Long getDurationMillis() {
+ return durationMillis;
+ }
+
+ public void setDurationMillis(Long durationMillis) {
+ this.durationMillis = durationMillis;
+ }
+
+ public byte[] getProfileValue() {
+ return profileValue;
+ }
+
+ public void setProfileValue(byte[] profileValue) {
+ this.profileValue = profileValue;
+ }
+
+ public void setProfileValue(Object profileValue) {
+ this.profileValue = SerDeUtils.toBytes(profileValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/GroupByPeriodFunction.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/GroupByPeriodFunction.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/GroupByPeriodFunction.java
new file mode 100644
index 0000000..1b602f4
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/GroupByPeriodFunction.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.metron.profiler.spark.function;
+
+import org.apache.metron.profiler.MessageRoute;
+import org.apache.metron.profiler.ProfilePeriod;
+import org.apache.spark.api.java.function.MapFunction;
+
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.PERIOD_DURATION;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.PERIOD_DURATION_UNITS;
+
+/**
+ * Defines how {@link MessageRoute} are grouped.
+ *
+ * The routes are grouped by (profile, entity, periodId) so that all of the required
+ * messages are available to produce a {@link org.apache.metron.profiler.ProfileMeasurement}.
+ */
+public class GroupByPeriodFunction implements MapFunction<MessageRoute, String> {
+
+ /**
+ * The duration of each profile period.
+ */
+ private int periodDuration;
+
+ /**
+ * The units of the period duration.
+ */
+ private TimeUnit periodDurationUnits;
+
+ public GroupByPeriodFunction(Properties profilerProperties) {
+ periodDurationUnits = TimeUnit.valueOf(PERIOD_DURATION_UNITS.get(profilerProperties, String.class));
+ periodDuration = PERIOD_DURATION.get(profilerProperties, Integer.class);
+ }
+
+ @Override
+ public String call(MessageRoute route) {
+ ProfilePeriod period = ProfilePeriod.fromTimestamp(route.getTimestamp(), periodDuration, periodDurationUnits);
+ return route.getProfileDefinition().getProfile() + "-" + route.getEntity() + "-" + period.getPeriod();
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/HBaseWriterFunction.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/HBaseWriterFunction.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/HBaseWriterFunction.java
new file mode 100644
index 0000000..cfabd94
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/HBaseWriterFunction.java
@@ -0,0 +1,171 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function;
+
+import org.apache.commons.collections4.IteratorUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.metron.hbase.HTableProvider;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.hbase.client.HBaseClient;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.hbase.ColumnBuilder;
+import org.apache.metron.profiler.hbase.RowKeyBuilder;
+import org.apache.metron.profiler.hbase.SaltyRowKeyBuilder;
+import org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder;
+import org.apache.metron.profiler.spark.ProfileMeasurementAdapter;
+import org.apache.spark.api.java.function.MapPartitionsFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_COLUMN_FAMILY;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_SALT_DIVISOR;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_TABLE_NAME;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_TABLE_PROVIDER;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_WRITE_DURABILITY;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.PERIOD_DURATION;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.PERIOD_DURATION_UNITS;
+
+/**
+ * Writes the profile measurements to HBase in Spark.
+ */
+public class HBaseWriterFunction implements MapPartitionsFunction<ProfileMeasurementAdapter, Integer> {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private TableProvider tableProvider;
+
+ /**
+ * The name of the HBase table to write to.
+ */
+ private String tableName;
+
+ /**
+ * The durability guarantee when writing to HBase.
+ */
+ private Durability durability;
+
+ /**
+ * Builds the HBase row key.
+ */
+ private RowKeyBuilder rowKeyBuilder;
+
+ /**
+ * Assembles the columns for HBase.
+ */
+ private ColumnBuilder columnBuilder;
+
+ public HBaseWriterFunction(Properties properties) {
+ tableName = HBASE_TABLE_NAME.get(properties, String.class);
+ durability = HBASE_WRITE_DURABILITY.get(properties, Durability.class);
+
+ // row key builder
+ int saltDivisor = HBASE_SALT_DIVISOR.get(properties, Integer.class);
+ int periodDuration = PERIOD_DURATION.get(properties, Integer.class);
+ TimeUnit periodDurationUnits = TimeUnit.valueOf(PERIOD_DURATION_UNITS.get(properties, String.class));
+ rowKeyBuilder = new SaltyRowKeyBuilder(saltDivisor, periodDuration, periodDurationUnits);
+
+ // column builder
+ String columnFamily = HBASE_COLUMN_FAMILY.get(properties, String.class);
+ columnBuilder = new ValueOnlyColumnBuilder(columnFamily);
+
+ // hbase table provider
+ String providerImpl = HBASE_TABLE_PROVIDER.get(properties, String.class);
+ tableProvider = createTableProvider(providerImpl);
+ }
+
+ /**
+ * Writes a set of measurements to HBase.
+ *
+ * @param iterator The measurements to write.
+ * @return The number of measurements written to HBase.
+ */
+ @Override
+ public Iterator<Integer> call(Iterator<ProfileMeasurementAdapter> iterator) throws Exception {
+ int count = 0;
+ LOG.debug("About to write profile measurement(s) to HBase");
+
+ // do not open hbase connection, if nothing to write
+ List<ProfileMeasurementAdapter> measurements = IteratorUtils.toList(iterator);
+ if(measurements.size() > 0) {
+
+ // open an HBase connection
+ Configuration config = HBaseConfiguration.create();
+ try (HBaseClient client = new HBaseClient(tableProvider, config, tableName)) {
+
+ for (ProfileMeasurementAdapter adapter : measurements) {
+ ProfileMeasurement m = adapter.toProfileMeasurement();
+ client.addMutation(rowKeyBuilder.rowKey(m), columnBuilder.columns(m), durability);
+ }
+ count = client.mutate();
+
+ } catch (IOException e) {
+ LOG.error("Unable to open connection to HBase", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ LOG.debug("{} profile measurement(s) written to HBase", count);
+ return IteratorUtils.singletonIterator(count);
+ }
+
+ /**
+ * Set the {@link TableProvider} using the class name of the provider.
+ * @param providerImpl The name of the class.
+ * @return
+ */
+ public HBaseWriterFunction withTableProviderImpl(String providerImpl) {
+ this.tableProvider = createTableProvider(providerImpl);
+ return this;
+ }
+
+ /**
+ * Creates a TableProvider based on a class name.
+ * @param providerImpl The class name of a TableProvider
+ */
+ private static TableProvider createTableProvider(String providerImpl) {
+ LOG.trace("Creating table provider; className={}", providerImpl);
+
+ // if class name not defined, use a reasonable default
+ if(StringUtils.isEmpty(providerImpl) || providerImpl.charAt(0) == '$') {
+ return new HTableProvider();
+ }
+
+ // instantiate the table provider
+ try {
+ Class<? extends TableProvider> clazz = (Class<? extends TableProvider>) Class.forName(providerImpl);
+ return clazz.getConstructor().newInstance();
+
+ } catch (InstantiationException | IllegalAccessException | IllegalStateException |
+ InvocationTargetException | NoSuchMethodException | ClassNotFoundException e) {
+ throw new IllegalStateException("Unable to instantiate connector", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java
new file mode 100644
index 0000000..cf8029f
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/MessageRouterFunction.java
@@ -0,0 +1,113 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function;
+
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.profiler.DefaultMessageRouter;
+import org.apache.metron.profiler.MessageRoute;
+import org.apache.metron.profiler.MessageRouter;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+/**
+ * The function responsible for finding routes for a given message in Spark.
+ */
+public class MessageRouterFunction implements FlatMapFunction<String, MessageRoute> {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * The global configuration used for the execution of Stellar.
+ */
+ private Map<String, String> globals;
+
+ /**
+ * The profile definitions.
+ */
+ private ProfilerConfig profilerConfig;
+
+ public MessageRouterFunction(ProfilerConfig profilerConfig, Map<String, String> globals) {
+ this.profilerConfig = profilerConfig;
+ this.globals = globals;
+ }
+
+ /**
+ * Find all routes for a given telemetry message.
+ *
+ * <p>A message may need routed to multiple profiles should it be needed by more than one. A
+ * message may also not be routed should it not be needed by any profiles.
+ *
+ * @param jsonMessage The raw JSON message.
+ * @return A list of message routes.
+ */
+ @Override
+ public Iterator<MessageRoute> call(String jsonMessage) throws Exception {
+ List<MessageRoute> routes;
+
+ JSONParser parser = new JSONParser();
+ Context context = TaskUtils.getContext(globals);
+ MessageRouter router = new DefaultMessageRouter(context);
+
+ // parse the raw message
+ Optional<JSONObject> message = toMessage(jsonMessage, parser);
+ if(message.isPresent()) {
+
+ // find all routes
+ routes = router.route(message.get(), profilerConfig, context);
+ LOG.trace("Found {} route(s) for a message", routes.size());
+
+ } else {
+ // the message is not valid and must be ignored
+ routes = Collections.emptyList();
+ LOG.trace("No route possible. Unable to parse message.");
+ }
+
+ return routes.iterator();
+ }
+
+ /**
+ * Parses the raw JSON of a message.
+ *
+ * @param json The raw JSON to parse.
+ * @param parser The parser to use.
+ * @return The parsed telemetry message.
+ */
+ private static Optional<JSONObject> toMessage(String json, JSONParser parser) {
+ try {
+ JSONObject message = (JSONObject) parser.parse(json);
+ return Optional.of(message);
+
+ } catch(Throwable e) {
+ LOG.warn(String.format("Unable to parse message, message will be ignored; message='%s'", json), e);
+ return Optional.empty();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/ProfileBuilderFunction.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/ProfileBuilderFunction.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/ProfileBuilderFunction.java
new file mode 100644
index 0000000..273695b
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/ProfileBuilderFunction.java
@@ -0,0 +1,107 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function;
+
+import org.apache.metron.profiler.DefaultMessageDistributor;
+import org.apache.metron.profiler.MessageDistributor;
+import org.apache.metron.profiler.MessageRoute;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.spark.ProfileMeasurementAdapter;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.spark.api.java.function.MapGroupsFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import static java.util.Comparator.comparing;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.PERIOD_DURATION;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.PERIOD_DURATION_UNITS;
+
+/**
+ * The function responsible for building profiles in Spark.
+ */
+public class ProfileBuilderFunction implements MapGroupsFunction<String, MessageRoute, ProfileMeasurementAdapter> {
+
+ protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private long periodDurationMillis;
+ private Map<String, String> globals;
+
+ public ProfileBuilderFunction(Properties properties, Map<String, String> globals) {
+ TimeUnit periodDurationUnits = TimeUnit.valueOf(PERIOD_DURATION_UNITS.get(properties, String.class));
+ int periodDuration = PERIOD_DURATION.get(properties, Integer.class);
+ this.periodDurationMillis = periodDurationUnits.toMillis(periodDuration);
+ this.globals = globals;
+ }
+
+ /**
+ * Build a profile from a set of message routes.
+ *
+ * <p>This assumes that all of the necessary routes have been provided
+ *
+ * @param group The group identifier.
+ * @param iterator The message routes.
+ * @return
+ */
+ @Override
+ public ProfileMeasurementAdapter call(String group, Iterator<MessageRoute> iterator) throws Exception {
+ // create the distributor; some settings are unnecessary because it is cleaned-up immediately after processing the batch
+ int maxRoutes = Integer.MAX_VALUE;
+ long profileTTLMillis = Long.MAX_VALUE;
+ MessageDistributor distributor = new DefaultMessageDistributor(periodDurationMillis, profileTTLMillis, maxRoutes);
+ Context context = TaskUtils.getContext(globals);
+
+ // sort the messages/routes
+ List<MessageRoute> routes = toStream(iterator)
+ .sorted(comparing(rt -> rt.getTimestamp()))
+ .collect(Collectors.toList());
+ LOG.debug("Building a profile for group '{}' from {} message(s)", group, routes.size());
+
+ // apply each message/route to build the profile
+ for(MessageRoute route: routes) {
+ distributor.distribute(route, context);
+ }
+
+ // flush the profile
+ List<ProfileMeasurement> measurements = distributor.flush();
+ if(measurements.size() > 1) {
+ throw new IllegalStateException("No more than 1 profile measurement is expected");
+ }
+
+ ProfileMeasurement m = measurements.get(0);
+ LOG.debug("Profile measurement created; profile={}, entity={}, period={}, value={}",
+ m.getProfileName(), m.getEntity(), m.getPeriod().getPeriod(), m.getProfileValue());
+ return new ProfileMeasurementAdapter(m);
+ }
+
+ private static <T> Stream<T> toStream(Iterator<T> iterator) {
+ Iterable<T> iterable = () -> iterator;
+ return StreamSupport.stream(iterable.spliterator(), false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/TaskUtils.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/TaskUtils.java b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/TaskUtils.java
new file mode 100644
index 0000000..d401f12
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/main/java/org/apache/metron/profiler/spark/function/TaskUtils.java
@@ -0,0 +1,41 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function;
+
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.StellarFunctions;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class TaskUtils implements Serializable {
+
+ /**
+ * Create the execution context for running Stellar.
+ */
+ public static Context getContext(Map<String, String> globals) {
+ Context context = new Context.Builder()
+ .with(Context.Capabilities.GLOBAL_CONFIG, () -> globals)
+ .with(Context.Capabilities.STELLAR_CONFIG, () -> globals)
+ .build();
+ StellarFunctions.initialize(context);
+ return context;
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
new file mode 100644
index 0000000..f560740
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/BatchProfilerIntegrationTest.java
@@ -0,0 +1,111 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.hbase.mock.MockHTable;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.SparkSession;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Properties;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_COLUMN_FAMILY;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_TABLE_NAME;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_TABLE_PROVIDER;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_FORMAT;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.TELEMETRY_INPUT_PATH;
+import static org.junit.Assert.assertEquals;
+
+public class BatchProfilerIntegrationTest {
+
+ private static SparkSession spark;
+ private MockHTable profilerTable;
+ private Properties profilerProperties;
+
+ @BeforeClass
+ public static void setupSpark() {
+ SparkConf conf = new SparkConf()
+ .setMaster("local")
+ .setAppName("BatchProfilerIntegrationTest")
+ .set("spark.sql.shuffle.partitions", "8");
+ spark = SparkSession
+ .builder()
+ .config(conf)
+ .getOrCreate();
+ }
+
+ @AfterClass
+ public static void tearDownSpark() {
+ if(spark != null) {
+ spark.close();
+ }
+ }
+
+ @Before
+ public void setup() {
+ profilerProperties = new Properties();
+
+ // define the source of the input telemetry
+ profilerProperties.put(TELEMETRY_INPUT_PATH.getKey(), "src/test/resources/telemetry.json");
+ profilerProperties.put(TELEMETRY_INPUT_FORMAT.getKey(), "text");
+
+ // define where the output will go
+ String tableName = HBASE_TABLE_NAME.get(profilerProperties, String.class);
+ String columnFamily = HBASE_COLUMN_FAMILY.get(profilerProperties, String.class);
+ profilerProperties.put(HBASE_TABLE_PROVIDER.getKey(), MockHBaseTableProvider.class.getName());
+
+ // create the mock hbase table
+ profilerTable = (MockHTable) MockHBaseTableProvider.addToCache(tableName, columnFamily);
+ }
+
+ @Test
+ public void testBatchProfiler() {
+
+ // run the batch profiler
+ BatchProfiler profiler = new BatchProfiler();
+ profiler.run(spark, profilerProperties, getGlobals(), getProfile());
+
+ List<Put> puts = profilerTable.getPutLog();
+ assertEquals(2, puts.size());
+ }
+
+
+ private ProfilerConfig getProfile() {
+ ProfileConfig profile = new ProfileConfig()
+ .withProfile("profile1")
+ .withForeach("ip_src_addr")
+ .withUpdate("count", "count + 1")
+ .withResult("count");
+ return new ProfilerConfig()
+ .withProfile(profile);
+ }
+
+ private Properties getGlobals() {
+ return new Properties();
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/HBaseWriterFunctionTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/HBaseWriterFunctionTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/HBaseWriterFunctionTest.java
new file mode 100644
index 0000000..55f3e21
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/HBaseWriterFunctionTest.java
@@ -0,0 +1,176 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function;
+
+import org.apache.commons.collections4.IteratorUtils;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.hbase.mock.MockHBaseTableProvider;
+import org.apache.metron.profiler.ProfileMeasurement;
+import org.apache.metron.profiler.spark.ProfileMeasurementAdapter;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_COLUMN_FAMILY;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.HBASE_TABLE_NAME;
+
+public class HBaseWriterFunctionTest {
+
+ Properties profilerProperties;
+
+ @Before
+ public void setup() {
+ profilerProperties = getProfilerProperties();
+
+ // create a mock table for HBase
+ String tableName = HBASE_TABLE_NAME.get(profilerProperties, String.class);
+ String columnFamily = HBASE_COLUMN_FAMILY.get(profilerProperties, String.class);
+ MockHBaseTableProvider.addToCache(tableName, columnFamily);
+ }
+
+ @Test
+ public void testWrite() throws Exception {
+
+ JSONObject message = getMessage();
+ String entity = (String) message.get("ip_src_addr");
+ long timestamp = (Long) message.get("timestamp");
+ ProfileConfig profile = getProfile();
+
+ // setup the profile measurements that will be written
+ List<ProfileMeasurementAdapter> measurements = createMeasurements(1, entity, timestamp, profile);
+
+ // setup the function to test
+ HBaseWriterFunction function = new HBaseWriterFunction(profilerProperties);
+ function.withTableProviderImpl(MockHBaseTableProvider.class.getName());
+
+ // write the measurements
+ Iterator<Integer> results = function.call(measurements.iterator());
+
+ // validate the result
+ List<Integer> counts = IteratorUtils.toList(results);
+ Assert.assertEquals(1, counts.size());
+ Assert.assertEquals(1, counts.get(0).intValue());
+ }
+
+ @Test
+ public void testWriteMany() throws Exception {
+
+ JSONObject message = getMessage();
+ String entity = (String) message.get("ip_src_addr");
+ long timestamp = (Long) message.get("timestamp");
+ ProfileConfig profile = getProfile();
+
+ // setup the profile measurements that will be written
+ List<ProfileMeasurementAdapter> measurements = createMeasurements(10, entity, timestamp, profile);
+
+ // setup the function to test
+ HBaseWriterFunction function = new HBaseWriterFunction(profilerProperties);
+ function.withTableProviderImpl(MockHBaseTableProvider.class.getName());
+
+ // write the measurements
+ Iterator<Integer> results = function.call(measurements.iterator());
+
+ // validate the result
+ List<Integer> counts = IteratorUtils.toList(results);
+ Assert.assertEquals(1, counts.size());
+ Assert.assertEquals(10, counts.get(0).intValue());
+ }
+
+ @Test
+ public void testWriteNone() throws Exception {
+
+ // there are no profile measurements to write
+ List<ProfileMeasurementAdapter> measurements = new ArrayList<>();
+
+ // setup the function to test
+ HBaseWriterFunction function = new HBaseWriterFunction(profilerProperties);
+ function.withTableProviderImpl(MockHBaseTableProvider.class.getName());
+
+ // write the measurements
+ Iterator<Integer> results = function.call(measurements.iterator());
+
+ // validate the result
+ List<Integer> counts = IteratorUtils.toList(results);
+ Assert.assertEquals(1, counts.size());
+ Assert.assertEquals(0, counts.get(0).intValue());
+ }
+
+ /**
+ * Create a list of measurements for testing.
+ *
+ * @param count The number of messages to create.
+ * @param entity The entity.
+ * @param timestamp The timestamp.
+ * @param profile The profile definition.
+ * @return
+ */
+ private List<ProfileMeasurementAdapter> createMeasurements(int count, String entity, long timestamp, ProfileConfig profile) {
+ List<ProfileMeasurementAdapter> measurements = new ArrayList<>();
+
+ for(int i=0; i<count; i++) {
+ ProfileMeasurement measurement = new ProfileMeasurement()
+ .withProfileName(profile.getProfile())
+ .withEntity(entity)
+ .withPeriod(timestamp, 15, TimeUnit.MINUTES);
+
+ // wrap the measurement using the adapter
+ measurements.add(new ProfileMeasurementAdapter(measurement));
+ }
+
+ return measurements;
+ }
+
+ /**
+ * Returns a telemetry message to use for testing.
+ */
+ private JSONObject getMessage() {
+ JSONObject message = new JSONObject();
+ message.put("ip_src_addr", "192.168.1.1");
+ message.put("status", "red");
+ message.put("timestamp", System.currentTimeMillis());
+ return message;
+ }
+
+ /**
+ * Returns profiler properties to use for testing.
+ */
+ private Properties getProfilerProperties() {
+ return new Properties();
+ }
+
+ /**
+ * Returns a profile definition to use for testing.
+ */
+ private ProfileConfig getProfile() {
+ return new ProfileConfig()
+ .withProfile("profile1")
+ .withForeach("ip_src_addr")
+ .withUpdate("count", "count + 1")
+ .withResult("count");
+
+ }
+}
[3/4] metron git commit: METRON-1707 Port Profiler to Spark
(nickwallen) closes apache/metron#1150
Posted by ni...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java
new file mode 100644
index 0000000..ceaa7cd
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/MessageRouterFunctionTest.java
@@ -0,0 +1,114 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function;
+
+import com.google.common.collect.Lists;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.common.configuration.profiler.ProfilerConfig;
+import org.apache.metron.profiler.MessageRoute;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tests the {@link MessageRouterFunction}.
+ */
+public class MessageRouterFunctionTest {
+
+ /**
+ * { "ip_src_addr": "192.168.1.22" }
+ */
+ @Multiline
+ private String goodMessage;
+
+ /**
+ * { "ip_src_addr": "192.168.1.22"
+ */
+ private String badMessage;
+
+ @Test
+ public void testFindRoutes() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(oneProfile(), getGlobals());
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(1, routes.size());
+ Assert.assertEquals("profile1", routes.get(0).getProfileDefinition().getProfile());
+ }
+
+ /**
+ * A bad or invalid message should return no routes.
+ */
+ @Test
+ public void testWithBadMessage() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(oneProfile(), getGlobals());
+ Iterator<MessageRoute> iter = function.call(badMessage);
+
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(0, routes.size());
+ }
+
+ @Test
+ public void testFindMultipleRoutes() throws Exception {
+ MessageRouterFunction function = new MessageRouterFunction(twoProfiles(), getGlobals());
+ Iterator<MessageRoute> iter = function.call(goodMessage);
+
+ List<MessageRoute> routes = Lists.newArrayList(iter);
+ Assert.assertEquals(2, routes.size());
+ Assert.assertEquals("profile1", routes.get(0).getProfileDefinition().getProfile());
+ Assert.assertEquals("profile2", routes.get(1).getProfileDefinition().getProfile());
+ }
+
+ private ProfilerConfig oneProfile() {
+ ProfileConfig profile = new ProfileConfig()
+ .withProfile("profile1")
+ .withForeach("ip_src_addr")
+ .withUpdate("count", "count + 1")
+ .withResult("count");
+
+ return new ProfilerConfig()
+ .withProfile(profile);
+ }
+
+ private ProfilerConfig twoProfiles() {
+ ProfileConfig profile1 = new ProfileConfig()
+ .withProfile("profile1")
+ .withForeach("ip_src_addr")
+ .withUpdate("count", "count + 1")
+ .withResult("count");
+ ProfileConfig profile2 = new ProfileConfig()
+ .withProfile("profile2")
+ .withForeach("ip_src_addr")
+ .withUpdate("count", "count + 1")
+ .withResult("count");
+ return new ProfilerConfig()
+ .withProfile(profile1)
+ .withProfile(profile2);
+ }
+
+ private Map<String, String> getGlobals() {
+ return Collections.emptyMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/ProfileBuilderFunctionTest.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/ProfileBuilderFunctionTest.java b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/ProfileBuilderFunctionTest.java
new file mode 100644
index 0000000..d5a4dba
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/java/org/apache/metron/profiler/spark/function/ProfileBuilderFunctionTest.java
@@ -0,0 +1,98 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.metron.profiler.spark.function;
+
+import org.apache.metron.common.configuration.profiler.ProfileConfig;
+import org.apache.metron.profiler.MessageRoute;
+import org.apache.metron.profiler.ProfilePeriod;
+import org.apache.metron.profiler.spark.ProfileMeasurementAdapter;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.PERIOD_DURATION;
+import static org.apache.metron.profiler.spark.BatchProfilerConfig.PERIOD_DURATION_UNITS;
+
+public class ProfileBuilderFunctionTest {
+
+ @Test
+ public void testBuildProfile() throws Exception {
+ // setup the message and profile
+ JSONObject message = getMessage();
+ String entity = "192.168.1.1";
+ long timestamp = (Long) message.get("timestamp");
+ ProfileConfig profile = getProfile();
+
+ // setup the route
+ MessageRoute route = new MessageRoute(profile, entity, message, timestamp);
+ List<MessageRoute> routes = new ArrayList();
+ routes.add(route);
+ routes.add(route);
+ routes.add(route);
+ Properties profilerProperties = getProfilerProperties();
+
+ // setup the period
+ int periodDuration = PERIOD_DURATION.get(profilerProperties, Integer.class);
+ TimeUnit periodDurationUnits = TimeUnit.valueOf(PERIOD_DURATION_UNITS.get(profilerProperties, String.class));
+ ProfilePeriod expectedPeriod = ProfilePeriod.fromTimestamp(timestamp, periodDuration, periodDurationUnits);
+
+ // build the profile
+ ProfileBuilderFunction function = new ProfileBuilderFunction(profilerProperties, getGlobals());
+ ProfileMeasurementAdapter measurement = function.call("profile1-192.168.1.1-0", routes.iterator());
+
+ // validate the measurement
+ Assert.assertEquals(entity, measurement.getEntity());
+ Assert.assertEquals(profile.getProfile(), measurement.getProfileName());
+ Assert.assertEquals(routes.size(), measurement.toProfileMeasurement().getProfileValue());
+ Assert.assertEquals(expectedPeriod.getPeriod(), (long) measurement.getPeriodId());
+ }
+
+ private JSONObject getMessage() {
+ JSONObject message = new JSONObject();
+ message.put("ip_src_addr", "192.168.1.1");
+ message.put("status", "red");
+ message.put("timestamp", System.currentTimeMillis());
+ return message;
+ }
+
+ private Properties getProfilerProperties() {
+ return new Properties();
+ }
+
+ private Map<String, String> getGlobals() {
+ return Collections.emptyMap();
+ }
+
+ private ProfileConfig getProfile() {
+ return new ProfileConfig()
+ .withProfile("profile1")
+ .withForeach("ip_src_addr")
+ .withUpdate("count", "count + 1")
+ .withResult("count");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/3bfbf018/metron-analytics/metron-profiler-spark/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler-spark/src/test/resources/log4j.properties b/metron-analytics/metron-profiler-spark/src/test/resources/log4j.properties
new file mode 100644
index 0000000..c3db0af
--- /dev/null
+++ b/metron-analytics/metron-profiler-spark/src/test/resources/log4j.properties
@@ -0,0 +1,31 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+# Root logger option
+log4j.rootLogger=ERROR, stdout
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+
+log4j.logger.org.apache.metron.profiler=ERROR
+log4j.logger.org.apache.metron.profiler.spark=ERROR
\ No newline at end of file