You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ma...@apache.org on 2017/06/01 21:41:37 UTC
[18/44] metron git commit: METRON-943 Create traffic connections
report in zeppelin (justinleet) closes apache/incubator-metron#573
METRON-943 Create traffic connections report in zeppelin (justinleet) closes apache/incubator-metron#573
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b3e7222f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b3e7222f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b3e7222f
Branch: refs/heads/Metron_0.4.0
Commit: b3e7222f1c4fb527ff3b18e8bb47383528d2eb57
Parents: e375936
Author: justinleet <ju...@gmail.com>
Authored: Fri May 12 08:49:28 2017 -0400
Committer: leet <le...@apache.org>
Committed: Fri May 12 08:49:28 2017 -0400
----------------------------------------------------------------------
metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec | 3 +++
.../config/zeppelin/metron/metron-connection-volume-report.json | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/b3e7222f/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index e851c7f..b49e0e6 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -271,6 +271,7 @@ This package installs the Metron Indexing files
%{metron_home}/config/zeppelin/metron/metron-yaf-telemetry.json
%{metron_home}/config/zeppelin/metron/metron-connection-report.json
%{metron_home}/config/zeppelin/metron/metron-ip-report.json
+%{metron_home}/config/zeppelin/metron/metron-connection-volume-report.json
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -400,6 +401,8 @@ This package installs the Metron Management UI %{metron_home}
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
%changelog
+* Tue May 9 2017 Apache Metron <de...@metron.apache.org> - 0.4.0
+- Add Zeppelin Connection Volume Report Dashboard
* Thu May 4 2017 Ryan Merriman <me...@gmail.com> - 0.4.0
- Added REST
* Tue May 2 2017 David Lyle <dl...@gmail.com> - 0.4.0
http://git-wip-us.apache.org/repos/asf/metron/blob/b3e7222f/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json b/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json
new file mode 100644
index 0000000..22765d4
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json
@@ -0,0 +1 @@
+{"paragraphs":[{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339150477_-816854736","id":"20170509-141230_1340330181","dateCreated":"2017-05-09T14:12:30+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:7251","text":"%spark.sql\n\n#\n# load the Yaf telemetry that has been archived by Metron\n#\ncreate temporary table yaf\n using org.apache.spark.sql.json\n options (path \"hdfs:///apps/metron/indexing/indexed/yaf\")","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:11+0000","dateStarted":"2017-05-09T14:20:10+0000","result":{"code":"SUCCESS","type":"TEXT","msg":""}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_14943391
84335_-2067041830","id":"20170509-141304_1479312597","dateCreated":"2017-05-09T14:13:04+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7322","text":"%md\n\n### Connection Volume (Source) - Yaf\n\nThe volume of connections made from source IPs\n\nThis IPs retrieved are given by an IPv4 CIDR block.","dateUpdated":"2017-05-09T14:22:02+0000","dateFinished":"2017-05-09T14:22:02+0000","dateStarted":"2017-05-09T14:22:02+0000","result":{"code":"SUCCESS","type":"HTML","msg":"<h3>Connection Volume (Source) - Yaf</h3>\n<p>The volume of connections made from source IPs</p>\n<p>This IPs retrieved are given by an IPv4 CIDR block.</p>\n"}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{"CIDR":"192.0.0.0/8"},"forms":{"CIDR":{"name":"CIDR","displayName":"CIDR","type":"input","defaultValue":"","hidden":false}}},"jobName":"paragraph_14943391
91894_-766575224","id":"20170509-141311_2132481247","dateCreated":"2017-05-09T14:13:11+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7398","text":"%spark\nimport org.apache.spark.sql.Row\nimport scala.concurrent.duration._\nimport java.util.concurrent.TimeUnit\nimport org.apache.commons.net.util.SubnetUtils\n\nval cidr = z.input(\"CIDR\").toString\n@transient val utils = new SubnetUtils(cidr)\nutils.setInclusiveHostCount(true)\n@transient val info = utils.getInfo\n\nval ipToLong = (ipAddress: String) => {\n val ipAddressInArray = ipAddress.split(\"\\\\.\")\n var result = 0L\n for (i <- 0 to ipAddressInArray.length-1) {\n\t val power = 3 - i\n\t val ip = Integer.parseInt(ipAddressInArray(i))\n\t result += ip * Math.pow(256.0, power.toDouble).toLong\n }\n result\n}\n\nval bcLow = sc.broadcast(ipToLong(info.getLowAddress))\nval bcHigh = sc.broadcast(ipToLong(info.getHighAddress))\nval bcIpToLong = sc.broadcast(ipToLong
)\n\nval results = sqlContext.sql(\ns\"\"\"SELECT\n ip_src_addr,\n COUNT(*) AS count\nFROM\n yaf\nGROUP BY ip_src_addr\nORDER BY ip_src_addr\n\"\"\").flatMap {\n case Row(ip_src_addr: String, count: Long) => {\n val longSrc = bcIpToLong.value(ip_src_addr)\n if(bcLow.value <= longSrc && longSrc <= bcHigh.value) {\n\t\t List(ip_src_addr + \"\\t\" + count)\n } else {\n List.empty[String]\n }\n }\n }.collect()\n\nprint(\"%table ip_src_addr\\tcount\\n\" + results.mkString(\"\\n\"))","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:21+0000","dateStarted":"2017-05-09T14:20:10+0000","result":{"code":"SUCCESS","type":"TABLE","msg":"ip_src_addr\tcount\n192.168.138.158\t111\n192.168.138.2\t1\n192.168.66.1\t89\n192.168.66.121\t69","comment":"","msgTable":[[{"key":"count","value":"192.168.138.158"},{"key":"count","value":"111"}],[{"value":"192.168.138.2"},{"value":"1"}],[{"value":"192.168.66.1"},{"value":"89"}],[{"val
ue":"192.168.66.121"},{"value":"69"}]],"columnNames":[{"name":"ip_src_addr","index":0,"aggr":"sum"},{"name":"count","index":1,"aggr":"sum"}],"rows":[["192.168.138.158","111"],["192.168.138.2","1"],["192.168.66.1","89"],["192.168.66.121","69"]]}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339578537_-194906756","id":"20170509-141938_486503393","dateCreated":"2017-05-09T14:19:38+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7794","text":"%md\n\n### Connection Volume (Destination) - Yaf\n\nThe volume of connections made to destination IPs.\n\nThis IPs retrieved are given by an IPv4 CIDR block.","dateUpdated":"2017-05-09T14:22:06+0000","dateFinished":"2017-05-09T14:22:06+0000","dateStarted":"2017-05-09T14:22:06+0000","result":{"code":"SUCCESS","type":"HTM
L","msg":"<h3>Connection Volume (Destination) - Yaf</h3>\n<p>The volume of connections made to destination IPs.</p>\n<p>This IPs retrieved are given by an IPv4 CIDR block.</p>\n"}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{"CIDR":"192.0.0.0/8"},"forms":{"CIDR":{"name":"CIDR","displayName":"CIDR","type":"input","defaultValue":"","hidden":false}}},"jobName":"paragraph_1494339202329_1284921236","id":"20170509-141322_1098639923","dateCreated":"2017-05-09T14:13:22+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7474","text":"%spark\nimport org.apache.spark.sql.Row\nimport scala.concurrent.duration._\nimport java.util.concurrent.TimeUnit\nimport org.apache.commons.net.util.SubnetUtils\n\nval cidr = z.input(\"CIDR\").toString\n@transient val utils = new SubnetUtils(cidr)\nutils.setInclusiveHostCo
unt(true)\n@transient val info = utils.getInfo\n\nval ipToLong = (ipAddress: String) => {\n val ipAddressInArray = ipAddress.split(\"\\\\.\")\n var result = 0L\n for (i <- 0 to ipAddressInArray.length-1) {\n\t val power = 3 - i\n\t val ip = Integer.parseInt(ipAddressInArray(i))\n\t result += ip * Math.pow(256.0, power.toDouble).toLong\n }\n result\n}\n\nval bcLow = sc.broadcast(ipToLong(info.getLowAddress))\nval bcHigh = sc.broadcast(ipToLong(info.getHighAddress))\nval bcIpToLong = sc.broadcast(ipToLong)\n\nval results = sqlContext.sql(\ns\"\"\"SELECT\n ip_dst_addr,\n COUNT(*) AS count\nFROM\n yaf\nGROUP BY ip_dst_addr\nORDER BY ip_dst_addr\n\"\"\").flatMap {\n case Row(ip_dst_addr: String, count: Long) => {\n val longDst = bcIpToLong.value(ip_dst_addr)\n if(bcLow.value <= longDst && longDst <= bcHigh.value) {\n\t\t List(ip_dst_addr + \"\\t\" + count)\n } else {\n List.empty[String]\n }\n }\n }.collect()\n
\nprint(\"%table ip_dst_addr\\tcount\\n\" + results.mkString(\"\\n\"))\n","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:31+0000","dateStarted":"2017-05-09T14:20:11+0000","result":{"code":"SUCCESS","type":"TABLE","msg":"ip_dst_addr\tcount\n192.168.138.158\t151\n192.168.138.2\t2\n192.168.66.1\t69\n192.168.66.121\t86","comment":"","msgTable":[[{"key":"count","value":"192.168.138.158"},{"key":"count","value":"151"}],[{"value":"192.168.138.2"},{"value":"2"}],[{"value":"192.168.66.1"},{"value":"69"}],[{"value":"192.168.66.121"},{"value":"86"}]],"columnNames":[{"name":"ip_dst_addr","index":0,"aggr":"sum"},{"name":"count","index":1,"aggr":"sum"}],"rows":[["192.168.138.158","151"],["192.168.138.2","2"],["192.168.66.1","69"],["192.168.66.121","86"]]}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339303735_1
424887757","id":"20170509-141503_898772342","dateCreated":"2017-05-09T14:15:03+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7557","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:31+0000","dateStarted":"2017-05-09T14:20:21+0000","result":{"code":"SUCCESS","type":"TEXT","msg":""}}],"name":"Metron - Connection Volume Report","id":"2CER9F199","angularObjects":{"2CET9UGAN:shared_process":[],"2CHC4B4TT:shared_process":[],"2CF24S5PD:shared_process":[],"2CF6W9QPU:shared_process":[],"2CJJ4RS82:shared_process":[],"2CGYCAYBG:shared_process":[]},"config":{"looknfeel":"simple"},"info":{}}
\ No newline at end of file