You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by om...@apache.org on 2015/12/08 07:37:58 UTC
[34/51] [partial] incubator-metron git commit: Initial import of code
from https://github.com/OpenSOC/opensoc at
ac0b00373f8f56dfae03a8109af5feb373ea598e.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/LancopeExampleOutput
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/LancopeExampleOutput b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/LancopeExampleOutput
new file mode 100644
index 0000000..b1bccf9
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/LancopeExampleOutput
@@ -0,0 +1,40 @@
+{"message":"<131>Jul 17 15:27:27 smc-01 StealthWatch[12365]: 2014-06-24T14:37:58Z 192.168.200.9 199.237.198.232 Critical Bad Host The host has been observed doing something bad to another host. Source Host is http (80/tcp) client to target.host.name (199.237.198.232)","@version":"1","@timestamp":"2014-07-17T15:24:32.217Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:35:00 smc-01 StealthWatch[12365]: 2014-07-17T15:34:30Z 10.201.3.83 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 92.64M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:32:05.934Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:35:00 smc-01 StealthWatch[12365]: 2014-07-17T15:34:30Z 10.201.3.145 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 45.2M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:32:05.935Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:35:00 smc-01 StealthWatch[12365]: 2014-07-17T15:34:30Z 10.201.3.50 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 41.46M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:32:05.936Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:42:01 smc-01 StealthWatch[12365]: 2014-07-17T15:42:00Z 10.10.101.24 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 39.37M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:39:05.976Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:56:01 smc-01 StealthWatch[12365]: 2014-07-17T15:55:00Z 0.0.100.0 0.0.0.0 Major ICMP Flood The source IP has sent an excessive number of ICMP packets in the last 5 minutes. Observed 262.4k pp5m. Policy maximum allows up to 100k pp5m.","@version":"1","@timestamp":"2014-07-17T15:53:05.995Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:56:01 smc-01 StealthWatch[12365]: 2014-07-17T15:55:00Z 0.0.88.0 0.0.0.0 Major High Total Traffic The total traffic inbound + outbound exceeds the acceptable total traffic values. Observed 16.26G bytes. Expected 4.17G bytes, tolerance of 50 allows up to 15.06G bytes.","@version":"1","@timestamp":"2014-07-17T15:53:05.996Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:57:01 smc-01 StealthWatch[12365]: 2014-07-17T15:56:30Z 10.201.3.50 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 42.49M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:54:05.984Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.40.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.55M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.992Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.30.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.47M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.995Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.20.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 40.48M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.995Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.201.3.83 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 96.74M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.992Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.100.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 32.95M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.997Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.90.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.52M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:06.000Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.80.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.51M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:06.002Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.70.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.49M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:06.002Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.110.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 32.92M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:05.997Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.60.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.49M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:06.003Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 15:59:01 smc-01 StealthWatch[12365]: 2014-07-17T15:58:30Z 10.50.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.48M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T15:56:06.004Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:06:01 smc-01 StealthWatch[12365]: 2014-07-17T16:05:00Z 10.10.101.46 0.0.0.0 Major New Flows Initiated The host has exceeded the acceptable total number of new flows initiated in a 5-minute period. ","@version":"1","@timestamp":"2014-07-17T16:03:06.046Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:06:01 smc-01 StealthWatch[12365]: 2014-07-17T16:05:00Z 10.10.101.46 0.0.0.0 Major Max Flows Initiated The host has initiated more than an acceptable maximum number of flows. ","@version":"1","@timestamp":"2014-07-17T16:03:06.046Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.110.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 33.01M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.146Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.100.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 33.03M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.147Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.90.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.59M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.148Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.80.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.58M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.157Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.70.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.56M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.157Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.60.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.56M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.158Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.50.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.55M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.160Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.30.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.55M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.173Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.201.3.83 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 96.82M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.173Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.20.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 40.55M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.174Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:20:00Z 10.110.10.254 10.120.80.254 Minor Worm Propagation The host has scanned and connected on a particular port across more than one subnet, and the host was previously scanned and connected to by a host for which the Worm Activity alarm has been raised. Worm propagated from Source Host using smb (445/tcp)","@version":"1","@timestamp":"2014-07-17T16:17:05.174Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:20:00Z 10.100.10.254 10.110.100.254 Minor Worm Propagation The host has scanned and connected on a particular port across more than one subnet, and the host was previously scanned and connected to by a host for which the Worm Activity alarm has been raised. Worm propagated from Source Host using smb (445/tcp)","@version":"1","@timestamp":"2014-07-17T16:17:05.174Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:20:00Z 10.90.10.254 10.100.100.254 Minor Worm Propagation The host has scanned and connected on a particular port across more than one subnet, and the host was previously scanned and connected to by a host for which the Worm Activity alarm has been raised. Worm propagated from Source Host using smb (445/tcp)","@version":"1","@timestamp":"2014-07-17T16:17:05.174Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:20:00Z 10.80.10.254 10.90.100.254 Minor Worm Propagation The host has scanned and connected on a particular port across more than one subnet, and the host was previously scanned and connected to by a host for which the Worm Activity alarm has been raised. Worm propagated from Source Host using smb (445/tcp)","@version":"1","@timestamp":"2014-07-17T16:17:05.175Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:20:00Z 10.70.10.254 10.80.100.254 Minor Worm Propagation The host has scanned and connected on a particular port across more than one subnet, and the host was previously scanned and connected to by a host for which the Worm Activity alarm has been raised. Worm propagated from Source Host using smb (445/tcp)","@version":"1","@timestamp":"2014-07-17T16:17:05.183Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:20:00Z 10.60.10.254 10.70.100.254 Minor Worm Propagation The host has scanned and connected on a particular port across more than one subnet, and the host was previously scanned and connected to by a host for which the Worm Activity alarm has been raised. Worm propagated from Source Host using smb (445/tcp)","@version":"1","@timestamp":"2014-07-17T16:17:05.184Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:20:00Z 10.50.10.254 10.60.100.254 Minor Worm Propagation The host has scanned and connected on a particular port across more than one subnet, and the host was previously scanned and connected to by a host for which the Worm Activity alarm has been raised. Worm propagated from Source Host using smb (445/tcp)","@version":"1","@timestamp":"2014-07-17T16:17:05.184Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:20:00Z 10.40.10.254 10.50.100.254 Minor Worm Propagation The host has scanned and connected on a particular port across more than one subnet, and the host was previously scanned and connected to by a host for which the Worm Activity alarm has been raised. Worm propagated from Source Host using smb (445/tcp)","@version":"1","@timestamp":"2014-07-17T16:17:05.184Z","type":"syslog","host":"10.122.196.201"}
+{"message":"<131>Jul 17 16:20:00 smc-01 StealthWatch[12365]: 2014-07-17T16:19:30Z 10.40.10.254 0.0.0.0 Minor High Concern Index The host's concern index has either exceeded the CI threshold or rapidly increased. Observed 36.63M points. Policy maximum allows up to 20M points.","@version":"1","@timestamp":"2014-07-17T16:17:05.168Z","type":"syslog","host":"192.249.113.37"}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/PCAPExampleOutput
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/PCAPExampleOutput b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/PCAPExampleOutput
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/SourcefireExampleOutput
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/SourcefireExampleOutput b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/SourcefireExampleOutput
new file mode 100644
index 0000000..5f177df
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/SampleInput/SourcefireExampleOutput
@@ -0,0 +1,2 @@
+SFIMS: [Primary Detection Engine (a7213248-6423-11e3-8537-fac6a92b7d9d)][MTD Access Control] Connection Type: Start, User: Unknown, Client: Unknown, Application Protocol: Unknown, Web App: Unknown, Firewall Rule Name: MTD Access Control, Firewall Rule Action: Allow, Firewall Rule Reasons: Unknown, URL Category: Unknown, URL_Reputation: Risk unknown, URL: Unknown, Interface Ingress: s1p1, Interface Egress: N/A, Security Zone Ingress: Unknown, Security Zone Egress: N/A, Security Intelligence Matching IP: None, Security Intelligence Category: None, {TCP} 72.163.0.129:60517 -> 10.1.128.236:443
+SFIMS: [Primary Detection Engine (a7213248-6423-11e3-8537-fac6a92b7d9d)][MTD Access Control] Connection Type: Start, User: Unknown, Client: Unknown, Application Protocol: Unknown, Web App: Unknown, Firewall Rule Name: MTD Access Control, Firewall Rule Action: Allow, Firewall Rule Reasons: Unknown, URL Category: Unknown, URL_Reputation: Risk unknown, URL: Unknown, Interface Ingress: s1p1, Interface Egress: N/A, Security Zone Ingress: Unknown, Security Zone Egress: N/A, Security Intelligence Matching IP: None, Security Intelligence Category: None, {TCP} 10.5.200.245:45283 -> 72.163.0.129:21
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf
new file mode 100644
index 0000000..dde089f
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Topologies/src/main/resources/TopologyConfigs_old/lancope.conf
@@ -0,0 +1,90 @@
+include = env/env_preprod.conf
+include = env/metrics.conf
+
+#Global Properties
+
+debug.mode=true
+local.mode=true
+num.workers=1
+
+#Simulator Spout
+spout.simulator.num.tasks=1
+spout.simulator.parallelism.hint=1
+
+#Kafka Spout
+spout.kafka.num.tasks=1
+spout.kafka.parallelism.hint=1
+spout.kafka.topic=test
+
+#Parser Bolt
+bolt.parser.num.tasks=1
+bolt.parser.parallelism.hint=1
+
+#Alerts Bolt (Static Configuration)
+bolt.alerts.num.tasks=1
+bolt.alerts.parallelism.hint=1
+bolt.alerts.staticpriority=3
+bolt.alerts.staticsource=lancope
+bolt.alerts.cluster=preprod
+
+#Host Enrichment
+bolt.enrichment.host.num.tasks=1
+bolt.enrichment.host.parallelism.hint=1
+bolt.enrichment.host.MAX_CACHE_SIZE=10000
+bolt.enrichment.host.MAX_TIME_RETAIN=10
+bolt.enrichment.host.enrichment_tag=host
+bolt.enrichment.host.source_ip=ip_src_addr
+bolt.enrichment.host.resp_ip=ip_dst_addr
+
+#GeoEnrichment
+bolt.enrichment.geo.num.tasks=1
+bolt.enrichment.geo.parallelism.hint=1
+bolt.enrichment.geo.enrichment_tag=geo
+bolt.enrichment.geo.source_ip=ip_src_addr
+bolt.enrichment.geo.resp_ip=ip_dst_addr
+bolt.enrichment.geo.adapter.table=GEO
+bolt.enrichment.geo.MAX_CACHE_SIZE=10000
+bolt.enrichment.geo.MAX_TIME_RETAIN=10
+
+#WhoisEnrichment
+bolt.enrichment.whois.num.tasks=1
+bolt.enrichment.whois.parallelism.hint=1
+bolt.enrichment.whois.whois_enrichment_tag=whois_enrichment
+bolt.enrichment.whois.source=host\":\"(.*?)\"
+bolt.enrichment.whois.MAX_CACHE_SIZE=10000
+bolt.enrichment.whois.MAX_TIME_RETAIN=10
+
+#CIF Enrichment
+bolt.enrichment.cif.tablename=cif_table
+bolt.enrichment.cif.num.tasks=1
+bolt.enrichment.cif.parallelism.hint=1
+bolt.enrichment.cif.source_ip=id.orig_h
+bolt.enrichment.cif.resp_ip=id.resp_h
+bolt.enrichment.cif.host=host
+bolt.enrichment.cif.email=email
+bolt.enrichment.cif.MAX_CACHE_SIZE=10000
+bolt.enrichment.cif.MAX_TIME_RETAIN=10
+
+
+#Indexing Bolt
+bolt.indexing.num.tasks=1
+bolt.indexing.parallelism.hint=1
+bolt.indexing.indexname=bro_index
+bolt.indexing.documentname=bro_doc
+bolt.indexing.bulk=200
+bolt.indexing.indexIP=ctrl01
+bolt.indexing.port=9200
+bolt.indexing.clustername=devo_es
+
+
+#HDFS Bolt
+bolt.hdfs.num.tasks=1
+bolt.hdfs.parallelism.hint=1
+bolt.hdfs.size.rotation.policy=5
+bolt.hdfs.size.sink.policy=5
+bolt.hdfs.fs.url=hdfs://nn1:8020
+
+#Kafka Bolt
+bolt.kafka.num.tasks=1
+bolt.kafka.parallelism.hint=1
+bolt.kafka.topic=test_out
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/opensoc-streaming/pom.xml b/opensoc-streaming/pom.xml
new file mode 100644
index 0000000..8f48583
--- /dev/null
+++ b/opensoc-streaming/pom.xml
@@ -0,0 +1,113 @@
+<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>com.opensoc</groupId>
+ <artifactId>OpenSOC-Streaming</artifactId>
+ <version>0.3BETA-SNAPSHOT</version>
+ <packaging>pom</packaging>
+ <name>OpenSOC-Streaming</name>
+ <description>Stream analytics for OpenSOC</description>
+ <url>www.getopensoc.com</url>
+ <properties>
+ <twitter>@ProjectOpenSOC</twitter>
+ <global_storm_version>0.9.2-incubating</global_storm_version>
+ <global_kafka_version>0.8.0</global_kafka_version>
+ <global_hadoop_version>2.2.0</global_hadoop_version>
+ <global_hbase_version>0.98.0-hadoop2</global_hbase_version>
+ <global_json_simple_version>1.1.1</global_json_simple_version>
+ <global_metrics_version>3.0.2</global_metrics_version>
+ <global_junit_version>4.4</global_junit_version>
+ <global_guava_version>18.0</global_guava_version>
+ <global_json_schema_validator_version>2.2.5</global_json_schema_validator_version>
+ </properties>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ <developers>
+ <developer>
+ <id>jsirota</id>
+ <name>James Sirota</name>
+ <email>jsirota@cisco.com</email>
+ <properties>
+ <twitter>@JamesSirota</twitter>
+ <blog>medium.com/@JamesSirota</blog>
+ </properties>
+ </developer>
+ </developers>
+
+
+ <modules>
+ <module>OpenSOC-Common</module>
+ <module>OpenSOC-EnrichmentAdapters</module>
+ <module>OpenSOC-MessageParsers</module>
+ <module>OpenSOC-Indexing</module>
+ <module>OpenSOC-Alerts</module>
+ <module>OpenSOC-DataLoads</module>
+ <module>OpenSOC-Topologies</module>
+ </modules>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.2</version>
+ </dependency>
+ </dependencies>
+ <build>
+
+ </build>
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemProperties>
+ <property>
+ <name>mode</name>
+ <value>local</value>
+ </property>
+ </systemProperties>
+ </configuration>
+ </plugin>
+ <!-- Normally, dependency report takes time, skip it -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.7</version>
+
+ <configuration>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <configuration>
+ <targetJdk>1.7</targetJdk>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>emma-maven-plugin</artifactId>
+ <version>1.0-alpha-3</version>
+ <inherited>true</inherited>
+ </plugin>
+ </plugins>
+ </reporting>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/pom.xml.versionsBackup
----------------------------------------------------------------------
diff --git a/opensoc-streaming/pom.xml.versionsBackup b/opensoc-streaming/pom.xml.versionsBackup
new file mode 100644
index 0000000..7302ae6
--- /dev/null
+++ b/opensoc-streaming/pom.xml.versionsBackup
@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>com.opensoc</groupId>
+ <artifactId>OpenSOC-Streaming</artifactId>
+ <version>BETA_0.2</version>
+ <packaging>pom</packaging>
+ <name>OpenSOC-Streaming</name>
+ <description>Stream analytics for OpenSOC</description>
+ <url>www.getopensoc.com</url>
+ <properties>
+ <twitter>@ProjectOpenSOC</twitter>
+ <global_version>BETA_0.2</global_version>
+ </properties>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ <developers>
+ <developer>
+ <id>jsirota</id>
+ <name>James Sirota</name>
+ <email>jsirota@cisco.com</email>
+ <properties>
+ <twitter>@JamesSirota</twitter>
+ <blog>medium.com/@JamesSirota</blog>
+ </properties>
+ </developer>
+ </developers>
+
+ <modules>
+ <module>OpenSOC-Common</module>
+ <module>OpenSOC-EnrichmentAdapters</module>
+ <module>OpenSOC-MessageParsers</module>
+ <module>OpenSOC-Indexing</module>
+ <module>OpenSOC-Alerts</module>
+ <module>OpenSOC-DataLoads</module>
+ <module>OpenSOC-Topologies</module>
+ </modules>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.2</version>
+ </dependency>
+ </dependencies>
+ <build>
+
+ </build>
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <systemProperties>
+ <property>
+ <name>mode</name>
+ <value>local</value>
+ </property>
+ </systemProperties>
+ </configuration>
+ </plugin>
+ <!-- Normally, dependency report takes time, skip it -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.7</version>
+
+ <configuration>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <configuration>
+ <targetJdk>1.7</targetJdk>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>emma-maven-plugin</artifactId>
+ <version>1.0-alpha-3</version>
+ <inherited>true</inherited>
+ </plugin>
+ </plugins>
+ </reporting>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/readme.md
----------------------------------------------------------------------
diff --git a/opensoc-streaming/readme.md b/opensoc-streaming/readme.md
new file mode 100644
index 0000000..d70667f
--- /dev/null
+++ b/opensoc-streaming/readme.md
@@ -0,0 +1,137 @@
+#Current Build
+
+The latest build of OpenSOC-Streaming is 0.3BETA. We are still in the process of merging/porting additional
+features from our production code base into this open source release. This release will be followed by
+a number of additional beta releases until the port is complete. We will also work on getting additional
+documentation and user/developer guides to the community as soon as we can. At this time we offer no support
+for the beta software, but will try to respond to requests as promptly as we can.
+
+# OpenSOC-Streaming
+
+Extensible set of Storm topologies and topology attributes for streaming, enriching, indexing, and storing telemetry in Hadoop. General information on OpenSOC is available at www.getopensoc.com
+
+For OpenSOC FAQ please read the following wiki entry: https://github.com/OpenSOC/opensoc-streaming/wiki/OpenSOC-FAQ
+
+
+# Usage Instructions
+
+## Message Parser Bolt
+
+Bolt for parsing telemetry messages into a JSON format
+
+```
+TelemetryParserBolt parser_bolt = new TelemetryParserBolt()
+ .withMessageParser(new BasicSourcefireParser())
+ .withOutputFieldName(topology_name);
+```
+
+###Parameters:
+
+MesageParser: parsers a raw message to JSON. Parsers listed below are available
+- BasicSourcefireParser: will parse a Sourcefire message to JSON
+- BasicBroParser: will parse a Bro message to JSON
+
+OutputFieldName: name of the output field emitted by the bolt
+
+## Telemetry Indexing Bolt
+
+Bolt for indexing JSON telemetry messages in ElasticSearch or Solr
+
+```
+TelemetryIndexingBolt indexing_bolt = new TelemetryIndexingBolt()
+ .withIndexIP(ElasticSearchIP).withIndexPort(elasticSearchPort)
+ .withClusterName(ElasticSearchClusterName)
+ .withIndexName(ElasticSearchIndexName)
+ .withDocumentName(ElasticSearchDocumentName).withBulk(bulk)
+ .withOutputFieldName(topology_name)
+ .withIndexAdapter(new ESBaseBulkAdapter());
+```
+
+###Parameters:
+
+IndexAdapter: adapter and strategy for indexing. Adapters listed below are available
+- ESBaseBulkAdapter: adapter for bulk loading telemetry into a single index in ElasticSearch
+- ESBulkRotatingAdapter: adapter for bulk loading telemetry into Elastic search, rotating once per hour, and applying a single alias to all rotated indexes
+- SolrAdapter (stubbed out, on roadmap)
+
+OutputFieldName: name of the output field emitted by the bolt
+
+IndexIP: IP of ElasticSearch/Solr
+
+IndexPort: Port of ElasticSearch/Solr
+
+ClusterName: ClusterName of ElasticSearch/Solr
+
+IndexName: IndexName of ElasticSearch/Solr
+
+DocumentName: DocumentName of ElasticSearch/Solr
+
+Bulk: number of documents to bulk load into ElasticSearch/Solr. If no value is passed, default is 10
+
+## Enrichment Bolt
+
+This bolt is for enriching telemetry messages with additional metadata from external data sources. At the time of the release the data sources supported are GeoIP (MaxMind GeoLite), WhoisDomain, Collective Intelligence Framework (CIF), and Lancope. In order to use the bolt the data sources have to be setup and data has to be bulk-loaded into them. The information on bulk-loading data sources and making them interoperable with the enrichment bolt is provided in the following wiki entries:
+
+- GeoIP: https://github.com/OpenSOC/opensoc-streaming/wiki/Setting-up-GeoLite-Data
+- WhoisDomain: https://github.com/OpenSOC/opensoc-streaming/wiki/Setting-up-Whois-Data
+- CIF Feeds: https://github.com/OpenSOC/opensoc-streaming/wiki/Setting-up-CIF-Data
+- Lancope Metadata: https://github.com/OpenSOC/opensoc-streaming/wiki/Setting-up-Lancope-data
+
+```
+Map<String, Pattern> patterns = new HashMap<String, Pattern>();
+ patterns.put("originator_ip_regex", Pattern.compile("ip_src_addr\":\"(.*?)\""));
+ patterns.put("responder_ip_regex", Pattern.compile("ip_dst_addr\":\"(.*?)\""));
+
+GeoMysqlAdapter geo_adapter = new GeoMysqlAdapter("IP", 0, "test", "test");
+
+GenericEnrichmentBolt geo_enrichment = new GenericEnrichmentBolt()
+ .withEnrichmentTag(geo_enrichment_tag)
+ .withOutputFieldName(topology_name).withAdapter(geo_adapter)
+ .withMaxTimeRetain(MAX_TIME_RETAIN)
+ .withMaxCacheSize(MAX_CACHE_SIZE).withPatterns(patterns);
+```
+
+###Parameters:
+
+GeoAdapter: adapter for the MaxMind GeoLite dataset. Adapters listed below are available
+- GeoMysqlAdapter: pulls geoIP data from MqSQL database
+- GeoPosgreSQLAdapter: pulls geoIP data from Posgress database (on road map, not yet available)
+
+WhoisAdapter: adapter for whois database. Adapters listed below are available
+- WhoisHBaseAdapter: adapter for HBase
+
+CIFAdapter: Hortonworks to document
+
+LancopeAdapter: Hortonworks to document
+
+originator_ip_regex: regex to extract the source ip form message
+
+responder_ip_regex: regex to extract dest ip from message
+The single bolt is currently undergoing testing and will be uploaded shortly
+
+geo_enrichment_tag: JSON field indicating how to tag the original message with the enrichment... {original_message:some_message, {geo_enrichment_tag:{from:xxx},{to:xxx}}}
+
+MAX_TIME_RETAIN: this bolt utilizes in-memory cache. this variable (in minutes) indicates now long to retain each entry in the cache
+
+MAX_CACHE_SIZE: this value defines the maximum size of the cache after which entries are evicted from cache
+
+OutputFieldName: name of the output field emitted by the bolt
+
+
+## Internal Test Spout
+
+We provide a capability to test a topology with messages stored in a file and packaged in a jar that is sent to storm. This functionality is exposed through a special spout that is able to replay test messages into a topology.
+
+```
+GenericInternalTestSpout test_spout = new GenericInternalTestSpout()
+ .withFilename("sourcefire_enriched").withRepeating(false)
+ .withMilisecondDelay(100);
+```
+
+###Parameters
+
+Filename: name of a file in a jar you want to replay
+
+Repeating: do you want to repeatedly play messages or stop after all the messages in the file have been read
+
+WithMilisecondDelay: the amount of the delay (sleep) between replayed messages
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/.gitignore
----------------------------------------------------------------------
diff --git a/opensoc-ui/.gitignore b/opensoc-ui/.gitignore
new file mode 100644
index 0000000..c94c2a1
--- /dev/null
+++ b/opensoc-ui/.gitignore
@@ -0,0 +1,39 @@
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+
+# Pcap files
+*.pcap
+
+# Config overrides
+config.json
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# Deployed apps should consider commenting this line out:
+# see https://npmjs.org/doc/faq.html#Should-I-check-my-node_modules-folder-into-git
+node_modules
+
+/.vagrant
+
+# Potentially sensitive seed data
+/seed/es
+/seed/*.pcap
+
+# temp files
+/tmp
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/.jshintignore
----------------------------------------------------------------------
diff --git a/opensoc-ui/.jshintignore b/opensoc-ui/.jshintignore
new file mode 100644
index 0000000..932c358
--- /dev/null
+++ b/opensoc-ui/.jshintignore
@@ -0,0 +1,3 @@
+lib/public
+coverage
+node_modules
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/.jshintrc
----------------------------------------------------------------------
diff --git a/opensoc-ui/.jshintrc b/opensoc-ui/.jshintrc
new file mode 100644
index 0000000..41630dc
--- /dev/null
+++ b/opensoc-ui/.jshintrc
@@ -0,0 +1,66 @@
+{
+ // See http://www.jshint.com/options/ for in-depth explanations
+
+ // Predefined globals that JSHint ignores
+ "browser" : true, // standard globals like 'window'
+ "devel" : true, // development globals, e.g. 'console'
+ "nonstandard" : true, // widely-adopted globals, e.g. 'escape'
+ "node" : true,
+ "jquery" : true,
+
+
+ "predef" : [ // extra globals
+ "angular",
+ "JST",
+ "MTD",
+ "google",
+
+ // Tests
+ "assert",
+ "sinon",
+ "describe",
+ "beforeEach",
+ "afterEach",
+ "loadFixtures",
+ "expect",
+ "before",
+ "after",
+ "it",
+ "mixpanel",
+ "nv",
+ "d3",
+
+ // stanford js crypto lib
+ "sjcl",
+
+ // Moment JS Date library
+ "moment",
+
+ // RequireJS
+ "requirejs",
+ "define",
+
+ // Angular global obj
+ "angular",
+
+ // Misc projects
+ "Presense",
+ "Refuge"
+ ],
+
+ // Development
+ "debug" : false, // warn about debugger statements
+
+ // Enforcing
+ "bitwise" : true, // prohibit the use of bitwise operations (slow and '&' is usually supposed to be '&&')
+ "curly" : true, // require {} for all blocks/scopes
+ "latedef" : true, // prohibit variable use before definition ("hoisting")
+ "noempty" : true, // prohibit empty blocks
+ "trailing" : true, // no trailing whitespace is allowed
+ "undef" : true, // prevent the use of undeclared variables
+
+ // Relaxing
+ "sub" : true, // allow all subscript notation, including '[]'
+ "laxcomma" : true, // allow commas after line breaks in lists
+ "strict" : false // don't force strict mode
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/.nodemonignore
----------------------------------------------------------------------
diff --git a/opensoc-ui/.nodemonignore b/opensoc-ui/.nodemonignore
new file mode 100644
index 0000000..fc7a8df
--- /dev/null
+++ b/opensoc-ui/.nodemonignore
@@ -0,0 +1,2 @@
+.vagrant
+node_modules
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/.travis.yml
----------------------------------------------------------------------
diff --git a/opensoc-ui/.travis.yml b/opensoc-ui/.travis.yml
new file mode 100644
index 0000000..d496e54
--- /dev/null
+++ b/opensoc-ui/.travis.yml
@@ -0,0 +1,19 @@
+language: node_js
+env: IN_TRAVIS=true
+node_js:
+- '0.10'
+notifications:
+ email:
+ recipients:
+ - opensoc-github@external.cisco.com
+ on_success: never
+ on_failure: always
+ hipchat:
+ rooms:
+ secure: ftMVn8V34kdqbwVUDMoKgbEKG4KzywcAIxByW0bUes18Fl4e0Tc5wUajfKCtB3ih9fazNgClQgoDZWhYGZ/Ik7o/DxwJdMd7SN36Vfl412LQiV7IVPO+vvVVvrAH5RxXA1yxQveNlF7DI6ANRwVISs/OsAplznIzmyqsJ/onn1I=
+
+addons:
+ postgresql: "9.3"
+before_script:
+ - psql -c 'create database opensoc_test;' -U postgres
+ - script/migrate up -e ci
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/Gruntfile.js
----------------------------------------------------------------------
diff --git a/opensoc-ui/Gruntfile.js b/opensoc-ui/Gruntfile.js
new file mode 100644
index 0000000..568611f
--- /dev/null
+++ b/opensoc-ui/Gruntfile.js
@@ -0,0 +1,29 @@
+module.exports = function (grunt) {
+ grunt.initConfig({
+ // copies frontend assets from bower_components into project
+ // bowercopy: {
+ // options: {
+ // clean: true
+ // },
+ // css: {
+ // options: {
+ // destPrefix: 'lib/public/css/vendor'
+ // },
+ // files: {
+ // 'bootstrap.css': 'bootstrap/dist/css/bootstrap.css',
+ // 'bootstrap-theme.css': 'bootstrap/dist/css/bootstrap-theme.css'
+ // }
+ // },
+ // libs: {
+ // options: {
+ // destPrefix: 'lib/public/js/vendor'
+ // },
+ // files: {
+ // 'angular.js': 'angular/angular.js'
+ // }
+ // }
+ // }
+ });
+
+ grunt.loadNpmTasks('grunt-bowercopy');
+};
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/LICENSE
----------------------------------------------------------------------
diff --git a/opensoc-ui/LICENSE b/opensoc-ui/LICENSE
new file mode 100644
index 0000000..ad410e1
--- /dev/null
+++ b/opensoc-ui/LICENSE
@@ -0,0 +1,201 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/Makefile
----------------------------------------------------------------------
diff --git a/opensoc-ui/Makefile b/opensoc-ui/Makefile
new file mode 100644
index 0000000..e268401
--- /dev/null
+++ b/opensoc-ui/Makefile
@@ -0,0 +1,29 @@
+testcmd=./node_modules/istanbul/lib/cli.js cover \
+ ./node_modules/mocha/bin/_mocha -- --check-leaks -R spec
+
+testwatchcmd=./node_modules/istanbul/lib/cli.js cover \
+ ./node_modules/mocha/bin/_mocha -- --check-leaks --watch -R spec
+
+test: test-all
+
+test-watch:
+ifeq ($(IN_TRAVIS),true)
+ PORT=4000 NODE_ENV=ci $(testwatchcmd)
+else
+ PORT=4000 NODE_ENV=test $(testwatchcmd)
+endif
+
+test-all:
+ifeq ($(IN_TRAVIS),true)
+ PORT=4000 NODE_ENV=ci $(testcmd)
+else
+ PORT=4000 NODE_ENV=test $(testcmd)
+endif
+
+# Load test data into DB
+seed:
+ node script/es_fetch.js && script/es_seed.sh
+
+clean:
+ rm -rf ./node_modules ./coverage
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/README.md
----------------------------------------------------------------------
diff --git a/opensoc-ui/README.md b/opensoc-ui/README.md
new file mode 100644
index 0000000..d01beb7
--- /dev/null
+++ b/opensoc-ui/README.md
@@ -0,0 +1,135 @@
+
+
+opensoc-ui
+==========
+
+User interface for OpenSOC
+
+## Deployment
+
+ Here are the minimal steps for deployment on a Ubuntu 14.04. These instructions will need to be altered for Ubuntu 12.04 as the nodejs package is too old. Assume that the code is in ```/opt/portal``` and the user is ```portal```.
+
+* Install dependencies:
+
+```bash
+apt-get update
+apt-get install -y libpcap-dev tshark redis-server nodejs npm
+ln -s /usr/bin/nodejs /usr/bin/node
+npm install -g pm2
+
+su - portal
+cd /opt/portal
+npm install --production
+```
+
+* Add a file name ```config.json``` to the repo root (```/opt/portal``` in our setup). The config should point to the various services. The following is an example config, all fields are required:
+
+```json
+{
+ "secret": "some secret",
+ "elasticsearch": {
+ "url": "http://192.168.33.10:9200"
+ },
+ "redis": {
+ "host": "127.0.0.1",
+ "port": 6379
+ },
+ "ldap": {
+ "url": "ldap://127.0.0.1:389",
+ "searchBase": "dc=opensoc,dc=dev",
+ "searchFilter": "(mail={{username}})",
+ "searchAttributes": ["cn", "uid", "mail", "givenName", "sn", "memberOf"],
+ "adminDn": "cn=admin,dc=opensoc,dc=dev",
+ "adminPassword": "opensoc"
+ },
+ "permissions": {
+ "pcap": "cn=investigators,ou=groups,dc=opensoc,dc=dev"
+ }
+ }
+```
+
+* Run the server:
+
+```bash
+pm2 start index.js -i max --name "opensoc"
+```
+
+
+## Setup development environment
+
+### Step 1: Install Virtualbox and Vagrant
+
+Download the latest package for your platform here:
+
+1. [Virtualbox](https://www.virtualbox.org/wiki/Downloads)
+2. [Vagrant](https://www.vagrantup.com/downloads.html)
+
+### Step 2: Clone repo
+
+```bash
+git clone git@github.com:OpenSOC/opensoc-ui.git
+cd opensoc-ui
+```
+
+### Step 3: Download and provision the development environment
+
+```bash
+vagrant up
+```
+
+You might see a couple warnings, but usually these can be ignored. Check for any obvious errors as this can cause problems running the portal later.
+
+### Step 4: SSH into the vm
+All dependencies will be installed in the VM. The repository root is shared between the host and VM. The shared volume is mounted at /vagrant. Use the following command to ssh into the newly built VM:
+
+```bash
+vagrant ssh
+cd vagrant
+```
+
+### Step 5: Seed the development VM
+
+To generate seed data for use with the opensoc-ui, use the following command.
+
+```bash
+script/es_gen.js
+```
+
+On the other hand, to duplicate another ES installation use:
+
+```bash
+ES_HOST=changeme.com script/es_fetch.js
+```
+
+You should now have seed data in ```seed/es```. You can load this into the dev ES instance with:
+
+```bash
+script/es_seed
+```
+
+For authentication, make sure you set up the LDAP directory structure with:
+
+```bash
+script/ldap_seed
+```
+
+### Step 6: Ensure tests pass
+
+You can now run the tests:
+
+```bash
+make test
+```
+
+### Step 7: Launch the server
+
+The ```nodemon``` utility automatically watches for changed files and reloads the node server automatically. Run the following commands from with the vagrant vm.
+
+```bash
+vagrant ssh
+cd /vagrant
+npm install -g nodemon
+nodemon
+```
+
+You can then access the OpenSOC ui at ```http://localhost:5000```.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/Vagrantfile
----------------------------------------------------------------------
diff --git a/opensoc-ui/Vagrantfile b/opensoc-ui/Vagrantfile
new file mode 100644
index 0000000..a2e5f99
--- /dev/null
+++ b/opensoc-ui/Vagrantfile
@@ -0,0 +1,129 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+ # All Vagrant configuration is done here. The most common configuration
+ # options are documented and commented below. For a complete reference,
+ # please see the online documentation at vagrantup.com.
+
+ # Every Vagrant virtual environment requires a box to build off of.
+ config.vm.box = "ubuntu/trusty64"
+
+ config.vm.provision "shell", path: 'script/provision'
+
+
+ # Nodemon server
+ config.vm.network :forwarded_port, guest: 5000, host: 5000
+
+ # Elasticsearch
+ config.vm.network :forwarded_port, guest: 9200, host: 9200
+
+ # Redis
+ # config.vm.network :forwarded_port, guest: 6379, host: 6379
+
+ # Disable automatic box update checking. If you disable this, then
+ # boxes will only be checked for updates when the user runs
+ # `vagrant box outdated`. This is not recommended.
+ # config.vm.box_check_update = false
+
+ # Create a private network, which allows host-only access to the machine
+ # using a specific IP.
+ config.vm.network "private_network", ip: "192.168.33.10"
+
+ # Create a public network, which generally matched to bridged network.
+ # Bridged networks make the machine appear as another physical device on
+ # your network.
+ # config.vm.network "public_network"
+
+ # If true, then any SSH connections made will enable agent forwarding.
+ # Default value: false
+ # config.ssh.forward_agent = true
+
+ # Share an additional folder to the guest VM. The first argument is
+ # the path on the host to the actual folder. The second argument is
+ # the path on the guest to mount the folder. And the optional third
+ # argument is a set of non-required options.
+ # config.vm.synced_folder "../data", "/vagrant_data"
+
+ # Provider-specific configuration so you can fine-tune various
+ # backing providers for Vagrant. These expose provider-specific options.
+ # Example for VirtualBox:
+ #
+ config.vm.provider "virtualbox" do |vb|
+ # # Don't boot with headless mode
+ # vb.gui = true
+ #
+ # # Use VBoxManage to customize the VM. For example to change memory:
+ vb.customize ["modifyvm", :id, "--memory", "2048"]
+ end
+ #
+ # View the documentation for the provider you're using for more
+ # information on available options.
+
+ # Enable provisioning with CFEngine. CFEngine Community packages are
+ # automatically installed. For example, configure the host as a
+ # policy server and optionally a policy file to run:
+ #
+ # config.vm.provision "cfengine" do |cf|
+ # cf.am_policy_hub = true
+ # # cf.run_file = "motd.cf"
+ # end
+ #
+ # You can also configure and bootstrap a client to an existing
+ # policy server:
+ #
+ # config.vm.provision "cfengine" do |cf|
+ # cf.policy_server_address = "10.0.2.15"
+ # end
+
+ # Enable provisioning with Puppet stand alone. Puppet manifests
+ # are contained in a directory path relative to this Vagrantfile.
+ # You will need to create the manifests directory and a manifest in
+ # the file default.pp in the manifests_path directory.
+ #
+ # config.vm.provision "puppet" do |puppet|
+ # puppet.manifests_path = "manifests"
+ # puppet.manifest_file = "site.pp"
+ # end
+
+ # Enable provisioning with chef solo, specifying a cookbooks path, roles
+ # path, and data_bags path (all relative to this Vagrantfile), and adding
+ # some recipes and/or roles.
+ #
+ # config.vm.provision "chef_solo" do |chef|
+ # chef.cookbooks_path = "../my-recipes/cookbooks"
+ # chef.roles_path = "../my-recipes/roles"
+ # chef.data_bags_path = "../my-recipes/data_bags"
+ # chef.add_recipe "mysql"
+ # chef.add_role "web"
+ #
+ # # You may also specify custom JSON attributes:
+ # chef.json = { :mysql_password => "foo" }
+ # end
+
+ # Enable provisioning with chef server, specifying the chef server URL,
+ # and the path to the validation key (relative to this Vagrantfile).
+ #
+ # The Opscode Platform uses HTTPS. Substitute your organization for
+ # ORGNAME in the URL and validation key.
+ #
+ # If you have your own Chef Server, use the appropriate URL, which may be
+ # HTTP instead of HTTPS depending on your configuration. Also change the
+ # validation key to validation.pem.
+ #
+ # config.vm.provision "chef_client" do |chef|
+ # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME"
+ # chef.validation_key_path = "ORGNAME-validator.pem"
+ # end
+ #
+ # If you're using the Opscode platform, your validator client is
+ # ORGNAME-validator, replacing ORGNAME with your organization name.
+ #
+ # If you have your own Chef Server, the default validation client name is
+ # chef-validator, unless you changed the configuration.
+ #
+ # chef.validation_client_name = "ORGNAME-validator"
+end
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/bower.json
----------------------------------------------------------------------
diff --git a/opensoc-ui/bower.json b/opensoc-ui/bower.json
new file mode 100644
index 0000000..f006a91
--- /dev/null
+++ b/opensoc-ui/bower.json
@@ -0,0 +1,24 @@
+{
+ "name": "opensoc-ui",
+ "main": "index.js",
+ "version": "0.0.0",
+ "homepage": "http://opensoc.github.io/opensoc-ui/",
+ "authors": [
+ "Jamil Bou Kheir <ja...@elbii.com>"
+ ],
+ "description": "OpenSOC Portal",
+ "moduleType": [
+ "node"
+ ],
+ "license": "Apache-2.0",
+ "private": true,
+ "ignore": [
+ "**/.*",
+ "node_modules",
+ "bower_components",
+ "test",
+ "tests"
+ ],
+ "dependencies": {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/doc/README.md
----------------------------------------------------------------------
diff --git a/opensoc-ui/doc/README.md b/opensoc-ui/doc/README.md
new file mode 100644
index 0000000..9231633
--- /dev/null
+++ b/opensoc-ui/doc/README.md
@@ -0,0 +1,4 @@
+Documentation
+=============
+
+## Welcome to OpenSOC!
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/examples/pcap-panel/pcap-parse.html
----------------------------------------------------------------------
diff --git a/opensoc-ui/examples/pcap-panel/pcap-parse.html b/opensoc-ui/examples/pcap-panel/pcap-parse.html
new file mode 100644
index 0000000..0e25dd8
--- /dev/null
+++ b/opensoc-ui/examples/pcap-panel/pcap-parse.html
@@ -0,0 +1,140 @@
+<html>
+ <head>
+ <script type="text/javascript" language="javascript" src="../../lib/public/vendor/jquery/jquery-1.8.0.js"></script>
+ <script type="text/javascript" language="javascript" src="../../seed/es/demo_packets.json"></script>
+
+ <style>
+ table {
+ border-collapse: collapse;
+ }
+ th {
+ text-align: left;
+ padding: 2px 10px 2px 5px;
+ }
+ td {
+ border-width: 1px;
+ padding: 6px;
+ border-style: inset;
+ border-color: gray;
+ background-color: white;}
+ td.info { min-width: 300px;}
+ </style>
+ </head>
+ <body>
+ <script type="text/javascript">
+ $(document).ready(function(e) {
+ /*
+ function print_packet_row ()
+ Number
+ Time
+ Source
+ Destination
+ Protocol
+ Length
+ Info
+
+
+ */
+
+ function print_packet_row ( packet_index ) {
+ var pcap_count = 0;
+
+ console.log(jsonObject.pdml.packet[packet_index].proto[0].field[0].$.name);
+
+ console.log(jsonObject.pdml.packet[packet_index].proto[0].field[0].$.name==="num");
+
+ var first_packet = jsonObject.pdml.packet[0];
+
+ var current_proto = jsonObject.pdml.packet[0].proto[0];
+
+ var temp_string = '<tr><td class="num"></td><td class="time"></td><td class="source"></td><td class="destination"></td><td class="protocols"></td><td class="length"></td><td class="info"></td></tr>';
+
+ var protos_length = first_packet.proto.length;
+ var current_proto_fields_length = current_proto.field.length;
+
+ for (i = 0; i < protos_length; ++i) {
+
+ console.log ('protos_length = ', protos_length, ', i = ', i);
+
+ current_protos_length = jsonObject.pdml.packet[packet_index].proto[i].field.length
+
+ for (j = 0; j < current_protos_length; ++j) {
+
+ console.log ('current_protos_length = ', current_protos_length, ', j = ', j, '\njsonObject.pdml.packet[packet_index].proto[i].field[j].$.show = ', jsonObject.pdml.packet[packet_index].proto[i].field[j].$.show, '\nname: ', jsonObject.pdml.packet[packet_index].proto[i].field[j].$.name);
+
+ switch ( jsonObject.pdml.packet[packet_index].proto[i].field[j].$.name ) {
+ case 'num':
+ // soon to be organized into an array to be placed in the dashboard
+ // currently swapping out the table cells via the DOM
+ temp_string = temp_string.replace('<td class="num">', '<td class="num">'+jsonObject.pdml.packet[packet_index].proto[i].field[j].$.show);
+ break;
+
+ case 'len':
+ // soon to be organized into an array to be placed in the dashboard
+ // currently swapping out the table cells via the DOM
+ temp_string = temp_string.replace('<td class="length">', '<td class="length">'+jsonObject.pdml.packet[packet_index].proto[i].field[j].$.show);
+ break;
+
+ case 'timestamp':
+ // soon to be organized into an array to be placed in the dashboard
+ // currently swapping out the table cells via the DOM
+ temp_string = temp_string.replace('<td class="time">', '<td class="time">'+jsonObject.pdml.packet[packet_index].proto[i].field[j].$.show);
+ break;
+
+ case 'eth.src':
+ // soon to be organized into an array to be placed in the dashboard
+ // currently swapping out the table cells via the DOM
+ temp_string = temp_string.replace('<td class="source">', '<td class="source">'+jsonObject.pdml.packet[packet_index].proto[i].field[j].$.show);
+ break;
+
+ case 'eth.dst':
+ // soon to be organized into an array to be placed in the dashboard
+ // currently swapping out the table cells via the DOM
+ temp_string = temp_string.replace('<td class="destination">', '<td class="destination">'+jsonObject.pdml.packet[packet_index].proto[i].field[j].$.show);
+ break;
+
+ case 'frame.protocols':
+ // soon to be organized into an array to be placed in the dashboard
+ // currently swapping out the table cells via the DOM
+ temp_string = temp_string.replace('<td class="protocols">', '<td class="protocols">'+jsonObject.pdml.packet[packet_index].proto[i].field[j].$.show);
+ break;
+ }
+
+ }
+
+ }
+
+ console.log(temp_string);
+ console.log('current_proto.field.length = ', current_proto.field.length);
+
+ $('table').append(temp_string);
+ }
+
+ $.each( jsonObject.pdml.packet, function ( index ) {
+ print_packet_row( index );
+ });
+
+
+ //console.log('jsonObject.pdml.packet.length = ', jsonObject.pdml.packet.length);
+
+
+
+ });
+ </script>
+ <dl>
+ <dt></dt>
+ <dd></dd>
+ </dl>
+ <table style="">
+ <tr>
+ <th>No.</th>
+ <th>Time</th>
+ <th>Source</th>
+ <th>Destination</th>
+ <th>Protocol</th>
+ <th>Length</th>
+ <th>Info</th>
+ </tr>
+ </table>
+ </body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/examples/server/nginx.conf
----------------------------------------------------------------------
diff --git a/opensoc-ui/examples/server/nginx.conf b/opensoc-ui/examples/server/nginx.conf
new file mode 100644
index 0000000..8700230
--- /dev/null
+++ b/opensoc-ui/examples/server/nginx.conf
@@ -0,0 +1,42 @@
+# Example OpenSOC configuration for nginx
+
+upstream opensoc {
+ server 127.0.0.1:5000;
+ keepalive 120;
+}
+
+server {
+ client_max_body_size 1g;
+
+ gzip on;
+ gzip_comp_level 6;
+ gzip_min_length 1000;
+ gzip_vary on;
+ gzip_proxied any;
+ gzip_types text/plain text/html application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+
+ listen 8080;
+
+ # Change to your host
+ server_name opensoc.dev;
+
+ # Change to your web root
+ root /var/www/opensoc-ui/lib/public;
+
+ location / {
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Host $http_host;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header X-NginX-Proxy true;
+ proxy_http_version 1.1;
+
+ # Required for HTML5 Websockets (Realtime Alerts)
+ proxy_set_header Connection "Upgrade";
+ proxy_set_header Upgrade $http_upgrade;
+
+ proxy_pass http://opensoc;
+ proxy_redirect off;
+ break;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/index.js
----------------------------------------------------------------------
diff --git a/opensoc-ui/index.js b/opensoc-ui/index.js
new file mode 100644
index 0000000..6f8309e
--- /dev/null
+++ b/opensoc-ui/index.js
@@ -0,0 +1 @@
+module.exports = require('./lib/opensoc-ui');
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/lib/modules/es-proxy.js
----------------------------------------------------------------------
diff --git a/opensoc-ui/lib/modules/es-proxy.js b/opensoc-ui/lib/modules/es-proxy.js
new file mode 100644
index 0000000..264b7b5
--- /dev/null
+++ b/opensoc-ui/lib/modules/es-proxy.js
@@ -0,0 +1,20 @@
+exports = module.exports = function(config) {
+ var httpProxy = require('http-proxy');
+ var proxy = httpProxy.createProxy();
+
+ proxy.on('error', function (err, req, res) {
+ console.log("[proxyError]", err);
+ });
+
+ return function(req, res, next) {
+ if (!req.user) {
+ res.send(403, 'Forbidden!');
+ return;
+ }
+
+ delete req.headers.cookie;
+ proxy.web(req, res, {
+ target: config.elasticsearch.url
+ });
+ };
+};
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/lib/modules/login.js
----------------------------------------------------------------------
diff --git a/opensoc-ui/lib/modules/login.js b/opensoc-ui/lib/modules/login.js
new file mode 100644
index 0000000..538af3e
--- /dev/null
+++ b/opensoc-ui/lib/modules/login.js
@@ -0,0 +1,32 @@
+exports = module.exports = function(app, config) {
+ var passport = require('passport');
+
+ app.get('/', function (req, res, next) {
+ if (!req.user) {
+ res.redirect('/login');
+ return;
+ }
+
+ res.render('index', {
+ user: JSON.stringify(req.user),
+ config: JSON.stringify({
+ elasticsearch: config.elasticsearch.url
+ })
+ });
+ });
+
+ app.get('/login', function (req, res) {
+ res.render('login', { flash: req.flash() });
+ });
+
+ app.post('/login', passport.authenticate('ldapauth', {
+ successRedirect: '/',
+ failureRedirect: '/login',
+ failureFlash: true
+ }));
+
+ app.get('/logout', function (req, res) {
+ req.logout();
+ res.redirect('/login');
+ });
+};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/lib/modules/pcap.js
----------------------------------------------------------------------
diff --git a/opensoc-ui/lib/modules/pcap.js b/opensoc-ui/lib/modules/pcap.js
new file mode 100644
index 0000000..c6091a3
--- /dev/null
+++ b/opensoc-ui/lib/modules/pcap.js
@@ -0,0 +1,95 @@
+function readRawBytes(size, transit) {
+ var buffer = new Buffer(size);
+ var bytesRead = 0;
+ var bytesLeft, dataLeft, len, leftOver;
+ var data, offset;
+
+ while (bytesRead < size) {
+ if (!data || offset >= data.length) {
+ offset = 0;
+ data = transit.shift();
+ }
+
+ bytesLeft = size - bytesRead;
+ dataLeft = data.length - offset;
+ len = bytesLeft < dataLeft ? bytesLeft : dataLeft;
+ data.copy(buffer, bytesRead, offset, offset + len);
+ bytesRead += len;
+ offset += len;
+ }
+
+ if (offset < data.length) {
+ dataLeft = data.length - offset;
+ leftOver = new Buffer(dataLeft);
+ data.copy(leftOver, 0, offset, offset + dataLeft);
+ transit.unshift(leftOver);
+ }
+
+ return buffer;
+}
+
+
+exports = module.exports = function(app, config) {
+ var _ = require("lodash");
+ var fs = require("fs");
+ var spawn = require('child_process').spawn;
+ var querystring = require('querystring');
+ var XmlStream = require('xml-stream');
+
+ // Mock pcap service for use in development
+ if (config.pcap.mock) {
+ app.get('/sample/pcap/:command', function(req, res) {
+ res.sendfile('/vagrant/seed/hbot.pcap');
+ });
+ }
+
+ app.get('/pcap/:command', function(req, res) {
+ if (!req.user || !req.user.permissions.pcap) {
+ res.send(403, 'Forbidden!');
+ return;
+ }
+
+ var transit = [];
+ var pcapUrl = config.pcap.url + '/' + req.param('command');
+ pcapUrl += '?' + querystring.stringify(req.query);
+
+ var curl = spawn('curl', ['-s', pcapUrl]);
+ var tshark = spawn('tshark', ['-i', '-', '-T', 'pdml']);
+ var xml = new XmlStream(tshark.stdout);
+
+ xml.collect('proto');
+ xml.collect('field');
+
+ curl.stdout.pipe(tshark.stdin);
+ curl.stdout.on('data', function (data) {
+ transit.push(data);
+ });
+
+ var npcaps = 0;
+ xml.on('end', function() {
+ res.end(']}');
+ curl.stdout.unpipe(tshark.stdin);
+ curl.kill('SIGKILL');
+ tshark.kill('SIGKILL');
+ });
+
+ xml.on('endElement: packet', function(packet) {
+ var psize = parseInt(packet.proto[0].$.size);
+
+ if (!npcaps) {
+ res.set('Content-Type', 'application/json');
+ res.write('{objects: [\n');
+ // skip global header
+ readRawBytes(24, transit);
+ } else {
+ res.write(',\n');
+ }
+
+ // skip packet header
+ readRawBytes(16, transit);
+ packet.hexdump = readRawBytes(psize, transit).toString('hex');
+ res.write(JSON.stringify(packet));
+ npcaps++;
+ });
+ });
+};
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/lib/opensoc-ui.js
----------------------------------------------------------------------
diff --git a/opensoc-ui/lib/opensoc-ui.js b/opensoc-ui/lib/opensoc-ui.js
new file mode 100644
index 0000000..f740523
--- /dev/null
+++ b/opensoc-ui/lib/opensoc-ui.js
@@ -0,0 +1,102 @@
+var _ = require('lodash');
+var http = require('http');
+var path = require('path');
+
+var express = require('express');
+
+var connect = require('connect');
+var flash = require('connect-flash');
+
+var cookieParser = require('cookie-parser');
+var bodyParser = require('body-parser');
+var cookieSession = require('cookie-session');
+
+var passport = require('passport');
+var ldapauth = require('passport-ldapauth');
+
+var esProxy = require('./modules/es-proxy');
+var login = require('./modules/login');
+var pcap = require('./modules/pcap');
+
+var app = express();
+var config = require('./config');
+
+
+try {
+ config = _.merge(config, require('../config'));
+ console.log('Loaded config overrides');
+} catch(err) {
+ console.log('No config overrides provided');
+}
+
+app.set('view engine', 'jade');
+app.set('views', path.join(__dirname, 'views/'));
+
+// Cookie middleware
+app.use(connect.logger('dev'));
+app.use(flash());
+app.use(cookieParser());
+app.use(cookieSession({
+ secret: config.secret,
+ cookie: {maxAge: 1 * 24 * 60 * 60 * 1000} // 1-day sessions
+}));
+
+app.use(passport.initialize());
+app.use(passport.session());
+
+app.use("/__es", esProxy(config));
+app.use(bodyParser.urlencoded({extended: true}));
+app.use(bodyParser.json());
+
+// LDAP integration
+passport.use(new ldapauth.Strategy({
+ usernameField: 'email',
+ passwordField: 'password',
+ server: config.ldap
+}, function (user, done) {
+ return done(null, user);
+}));
+
+
+// Serialize LDAP user into session.
+passport.serializeUser(function (ldapUser, done) {
+ // ensure that memberOf is an array.
+ var memberOf = ldapUser.memberOf || [];
+ memberOf = _.isArray(memberOf) ? memberOf : [memberOf];
+ ldapUser.memberOf = memberOf;
+
+ // LDAP permissions
+ ldapUser.permissions = {};
+ var permissions = _.keys(config.permissions);
+ _.each(permissions, function (perm) {
+ var group = config.permissions[perm];
+ ldapUser.permissions[perm] = _.contains(memberOf, group);
+ });
+
+ done(null, JSON.stringify(ldapUser));
+});
+
+
+// De-serialize user from session.
+passport.deserializeUser(function (ldapUser, done) {
+ try {
+ done(null, JSON.parse(ldapUser));
+ } catch(err) {
+ done(null, null);
+ }
+});
+
+
+// Setup routes
+pcap(app, config);
+login(app, config);
+
+// Serve static assets
+app.use(connect.static(path.join(__dirname, 'public')));
+
+
+// Start server
+var server = http.createServer(app);
+server.listen(config.port || 5000);
+
+exports.app = app;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/lib/public/app/app.js
----------------------------------------------------------------------
diff --git a/opensoc-ui/lib/public/app/app.js b/opensoc-ui/lib/public/app/app.js
new file mode 100755
index 0000000..fa03b4a
--- /dev/null
+++ b/opensoc-ui/lib/public/app/app.js
@@ -0,0 +1,158 @@
+/**
+ * main app level module
+ */
+define([
+ 'angular',
+ 'jquery',
+ 'lodash',
+ 'require',
+ 'elasticjs',
+ 'bootstrap',
+ 'angular-sanitize',
+ 'angular-strap',
+ 'angular-dragdrop',
+ 'angular-cookies',
+ 'extend-jquery',
+ 'bindonce',
+],
+function (angular, $, _, appLevelRequire) {
+
+ "use strict";
+
+ var app = angular.module('kibana', []),
+ // we will keep a reference to each module defined before boot, so that we can
+ // go back and allow it to define new features later. Once we boot, this will be false
+ pre_boot_modules = [],
+ // these are the functions that we need to call to register different
+ // features if we define them after boot time
+ register_fns = {};
+
+ // This stores the Kibana revision number, @REV@ is replaced by grunt.
+ app.constant('kbnVersion',"@REV@");
+
+ // The minimum version that must be in the cluster
+ app.constant('esMinVersion','0.90.9');
+
+ // Use this for cache busting partials
+ app.constant('cacheBust',"cache-bust="+Date.now());
+
+ /**
+ * Tells the application to watch the module, once bootstraping has completed
+ * the modules controller, service, etc. functions will be overwritten to register directly
+ * with this application.
+ * @param {[type]} module [description]
+ * @return {[type]} [description]
+ */
+ app.useModule = function (module) {
+ if (pre_boot_modules) {
+ pre_boot_modules.push(module);
+ } else {
+ _.extend(module, register_fns);
+ }
+ return module;
+ };
+
+ app.safeApply = function ($scope, fn) {
+ switch($scope.$$phase) {
+ case '$apply':
+ // $digest hasn't started, we should be good
+ $scope.$eval(fn);
+ break;
+ case '$digest':
+ // waiting to $apply the changes
+ setTimeout(function () { app.safeApply($scope, fn); }, 10);
+ break;
+ default:
+ // clear to begin an $apply $$phase
+ $scope.$apply(fn);
+ break;
+ }
+ };
+
+ app.config(function ($routeProvider, $controllerProvider, $compileProvider, $filterProvider, $provide) {
+
+ $routeProvider
+ .when('/dashboard', {
+ templateUrl: 'app/partials/dashboard.html',
+ })
+ .when('/dashboard/:kbnType/:kbnId', {
+ templateUrl: 'app/partials/dashboard.html',
+ })
+ .when('/dashboard/:kbnType/:kbnId/:params', {
+ templateUrl: 'app/partials/dashboard.html'
+ })
+ .otherwise({
+ redirectTo: 'dashboard'
+ });
+
+ // this is how the internet told me to dynamically add modules :/
+ register_fns.controller = $controllerProvider.register;
+ register_fns.directive = $compileProvider.directive;
+ register_fns.factory = $provide.factory;
+ register_fns.service = $provide.service;
+ register_fns.filter = $filterProvider.register;
+ });
+
+ var apps_deps = [
+ 'elasticjs.service',
+ '$strap.directives',
+ 'ngSanitize',
+ 'ngDragDrop',
+ 'ngCookies',
+ 'kibana',
+ 'pasvaz.bindonce'
+ ];
+
+ _.each('controllers directives factories services filters'.split(' '),
+ function (type) {
+ var module_name = 'kibana.'+type;
+ // create the module
+ app.useModule(angular.module(module_name, []));
+ // push it into the apps dependencies
+ apps_deps.push(module_name);
+ });
+
+ app.panel_helpers = {
+ partial: function (name) {
+ return 'app/partials/'+name+'.html';
+ }
+ };
+
+ // load the core components
+ require([
+ 'controllers/all',
+ 'directives/all',
+ 'filters/all'
+ ], function () {
+
+ // bootstrap the app
+ angular
+ .element(document)
+ .ready(function() {
+ $('html').attr('ng-controller', 'DashCtrl');
+ angular.bootstrap(document, apps_deps)
+ .invoke(['$rootScope', function ($rootScope) {
+ _.each(pre_boot_modules, function (module) {
+ _.extend(module, register_fns);
+ });
+ pre_boot_modules = false;
+
+ $rootScope.requireContext = appLevelRequire;
+ $rootScope.require = function (deps, fn) {
+ var $scope = this;
+ $scope.requireContext(deps, function () {
+ var deps = _.toArray(arguments);
+ // Check that this is a valid scope.
+ if($scope.$id) {
+ $scope.$apply(function () {
+ fn.apply($scope, deps);
+ });
+ }
+ });
+ };
+ }]);
+ });
+ });
+
+ return app;
+});
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-ui/lib/public/app/components/extend-jquery.js
----------------------------------------------------------------------
diff --git a/opensoc-ui/lib/public/app/components/extend-jquery.js b/opensoc-ui/lib/public/app/components/extend-jquery.js
new file mode 100755
index 0000000..7d81aa4
--- /dev/null
+++ b/opensoc-ui/lib/public/app/components/extend-jquery.js
@@ -0,0 +1,47 @@
+define(['jquery'],
+function ($) {
+ 'use strict';
+
+ /**
+ * jQuery extensions
+ */
+ var $win = $(window);
+
+ $.fn.place_tt = (function () {
+ var defaults = {
+ offset: 5,
+ css: {
+ position : 'absolute',
+ top : -1000,
+ left : 0,
+ color : "#c8c8c8",
+ padding : '10px',
+ 'font-size': '11pt',
+ 'font-weight' : 200,
+ 'background-color': '#1f1f1f',
+ 'border-radius': '5px',
+ 'z-index': 9999
+ }
+ };
+
+ return function (x, y, opts) {
+ opts = $.extend(true, {}, defaults, opts);
+ return this.each(function () {
+ var $tooltip = $(this), width, height;
+
+ $tooltip.css(opts.css);
+ if (!$.contains(document.body, $tooltip[0])) {
+ $tooltip.appendTo(document.body);
+ }
+
+ width = $tooltip.outerWidth(true);
+ height = $tooltip.outerHeight(true);
+
+ $tooltip.css('left', x + opts.offset + width > $win.width() ? x - opts.offset - width : x + opts.offset);
+ $tooltip.css('top', y + opts.offset + height > $win.height() ? y - opts.offset - height : y + opts.offset);
+ });
+ };
+ })();
+
+ return $;
+});