You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by om...@apache.org on 2015/12/08 07:38:15 UTC
[51/51] [partial] incubator-metron git commit: Initial import of code
from https://github.com/OpenSOC/opensoc at
ac0b00373f8f56dfae03a8109af5feb373ea598e.
Initial import of code from https://github.com/OpenSOC/opensoc at ac0b00373f8f56dfae03a8109af5feb373ea598e.
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/05e188ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/05e188ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/05e188ba
Branch: refs/heads/master
Commit: 05e188ba209d423b4489fbea2c4bdb3362ad62ed
Parents:
Author: Owen O'Malley <om...@apache.org>
Authored: Mon Dec 7 22:29:00 2015 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Mon Dec 7 22:29:00 2015 -0800
----------------------------------------------------------------------
LICENSE | 201 +
README.md | 61 +
opensoc-streaming/.gitignore | 15 +
opensoc-streaming/.travis.yml | 14 +
opensoc-streaming/LICENSE | 201 +
opensoc-streaming/OpenSOC-Alerts/pom.xml | 81 +
.../OpenSOC-Alerts/pom.xml.versionsBackup | 99 +
.../com/opensoc/alerts/AbstractAlertBolt.java | 117 +
.../com/opensoc/alerts/TelemetryAlertsBolt.java | 256 +
.../alerts/adapters/AbstractAlertAdapter.java | 70 +
.../alerts/adapters/AllAlertAdapter.java | 223 +
.../adapters/HbaseWhiteAndBlacklistAdapter.java | 420 +
.../com/opensoc/tagging/AbstractTaggerBolt.java | 93 +
.../opensoc/tagging/TelemetryTaggerBolt.java | 199 +
.../tagging/adapters/AbstractTaggerAdapter.java | 34 +
.../opensoc/tagging/adapters/RegexTagger.java | 64 +
.../tagging/adapters/StaticAllTagger.java | 53 +
.../opensoc/tagging/adapters/TaggerAdapter.java | 9 +
.../src/main/resources/hbase-site.xml | 90 +
opensoc-streaming/OpenSOC-Common/pom.xml | 132 +
.../OpenSOC-Common/pom.xml.versionsBackup | 131 +
.../alerts/interfaces/AlertsAdapter.java | 16 +
.../alerts/interfaces/AlertsInterface.java | 11 +
.../alerts/interfaces/TaggerAdapter.java | 9 +
.../interfaces/EnrichmentAdapter.java | 26 +
.../main/java/com/opensoc/hbase/HBaseBolt.java | 126 +
.../opensoc/hbase/HBaseStreamPartitioner.java | 146 +
.../java/com/opensoc/hbase/HTableConnector.java | 106 +
.../com/opensoc/hbase/TupleTableConfig.java | 279 +
.../opensoc/index/interfaces/IndexAdapter.java | 11 +
.../java/com/opensoc/ise/parser/ISEParser.java | 629 +
.../java/com/opensoc/ise/parser/ISEParser.jj | 12 +
.../opensoc/ise/parser/ISEParserConstants.java | 52 +
.../ise/parser/ISEParserTokenManager.java | 662 +
.../com/opensoc/ise/parser/JavaCharStream.java | 616 +
.../com/opensoc/ise/parser/ParseException.java | 187 +
.../main/java/com/opensoc/ise/parser/Token.java | 131 +
.../com/opensoc/ise/parser/TokenMgrError.java | 147 +
.../json/serialization/JSONDecoderHelper.java | 111 +
.../json/serialization/JSONEncoderHelper.java | 87 +
.../json/serialization/JSONKafkaSerializer.java | 263 +
.../json/serialization/JSONKryoSerializer.java | 55 +
.../com/opensoc/metrics/MetricReporter.java | 89 +
.../com/opensoc/metrics/MyMetricReporter.java | 33 +
.../java/com/opensoc/metrics/NullReporter.java | 10 +
.../parser/interfaces/MessageFilter.java | 10 +
.../parser/interfaces/MessageParser.java | 10 +
.../main/java/com/opensoc/pcap/Constants.java | 21 +
.../main/java/com/opensoc/pcap/IEEE_802_1Q.java | 27 +
.../opensoc/pcap/OpenSocEthernetDecoder.java | 117 +
.../main/java/com/opensoc/pcap/PacketInfo.java | 401 +
.../com/opensoc/pcap/PcapByteInputStream.java | 168 +
.../main/java/com/opensoc/pcap/PcapUtils.java | 394 +
.../src/main/java/com/opensoc/pcap/asdf.java | 5 +
.../com/opensoc/test/AbstractTestContext.java | 189 +
.../java/com/opensoc/test/ISEParserTest.java | 27 +
.../java/com/opensoc/topologyhelpers/Cli.java | 186 +
.../opensoc/topologyhelpers/ErrorGenerator.java | 27 +
.../opensoc/topologyhelpers/SettingsLoader.java | 118 +
.../dependency-reduced-pom.xml | 145 +
opensoc-streaming/OpenSOC-DataLoads/pom.xml | 90 +
.../OpenSOC-DataLoads/pom.xml.versionsBackup | 97 +
.../opensoc/dataloads/cif/HBaseTableLoad.java | 122 +
.../OpenSOC-DataLoads/src/hbase-site.xml | 100 +
.../OpenSOC-DataServices/conf/config.properties | 21 +
opensoc-streaming/OpenSOC-DataServices/pom.xml | 278 +
.../alerts/server/AlertsCacheReaper.java | 45 +
.../alerts/server/AlertsFilterCacheEntry.java | 17 +
.../alerts/server/AlertsProcessingServer.java | 44 +
.../opensoc/alerts/server/AlertsSearcher.java | 237 +
.../java/com/opensoc/dataservices/Main.java | 288 +
.../opensoc/dataservices/auth/AuthToken.java | 187 +
.../dataservices/auth/AuthTokenFilter.java | 15 +
.../dataservices/auth/CustomDomainADRealm.java | 34 +
.../auth/RestSecurityInterceptor.java | 57 +
.../dataservices/common/OpenSOCService.java | 27 +
.../dataservices/kafkaclient/KafkaClient.java | 83 +
.../dataservices/kafkaclient/KafkaConsumer.java | 49 +
.../kafkaclient/poll/PollingKafkaClient.java | 101 +
.../kafkaclient/poll/PollingKafkaConsumer.java | 52 +
.../modules/guice/AlertsServerModule.java | 36 +
.../modules/guice/DefaultServletModule.java | 48 +
.../modules/guice/DefaultShiroWebModule.java | 90 +
.../modules/guice/RestEasyModule.java | 23 +
.../modules/guice/ServiceModule.java | 34 +
.../com/opensoc/dataservices/rest/Index.java | 53 +
.../opensoc/dataservices/rest/RestServices.java | 35 +
.../dataservices/servlet/LoginServlet.java | 113 +
.../dataservices/servlet/LogoutServlet.java | 51 +
.../websocket/KafkaMessageSenderServlet.java | 27 +
.../websocket/KafkaMessageSenderSocket.java | 122 +
.../websocket/KafkaWebSocketCreator.java | 60 +
.../ElasticSearch_KafkaAlertsService.java | 89 +
.../alerts/Solr_KafkaAlertsService.java | 89 +
.../src/main/resources/log4j.xml | 17 +
.../main/resources/webroot/WEB-INF/shiro.ini | 45 +
.../src/main/resources/webroot/include.jsp | 20 +
.../src/main/resources/webroot/logged_in.jsp | 22 +
.../src/main/resources/webroot/login.jsp | 70 +
.../src/main/resources/webroot/login_failed.jsp | 14 +
.../src/main/resources/webroot/login_old.jsp | 21 +
.../src/main/resources/webroot/withsocket.jsp | 100 +
.../src/main/resources/webroot/withsocket2.jsp | 73 +
.../src/main/resources/webroot/withsocket3.jsp | 100 +
.../OpenSOC-EnrichmentAdapters/pom.xml | 133 +
.../pom.xml.versionsBackup | 136 +
.../adapters/cif/AbstractCIFAdapter.java | 41 +
.../adapters/cif/CIFHbaseAdapter.java | 125 +
.../adapters/geo/AbstractGeoAdapter.java | 61 +
.../adapters/geo/GeoMysqlAdapter.java | 185 +
.../adapters/host/AbstractHostAdapter.java | 40 +
.../host/HostFromPropertiesFileAdapter.java | 56 +
.../adapters/whois/AbstractWhoisAdapter.java | 40 +
.../adapters/whois/WhoisHBaseAdapter.java | 119 +
.../common/AbstractEnrichmentBolt.java | 128 +
.../common/GenericEnrichmentBolt.java | 274 +
.../src/main/resources/hbase-site.xml | 90 +
.../adapters/cif/CIFHbaseAdapterTest.java | 197 +
.../adapters/geo/GeoMysqlAdapterTest.java | 153 +
.../adapters/whois/WhoisHBaseAdapterTest.java | 137 +
.../resources/CIFHbaseAdapterTest.properties | 11 +
.../resources/GeoMysqlAdapterTest.properties | 11 +
.../resources/WhoisHbaseAdapterTest.properties | 11 +
.../OpenSOC-FlumeConfigs/SampleFlumeConf.rtf | 43 +
opensoc-streaming/OpenSOC-Indexing/pom.xml | 89 +
.../OpenSOC-Indexing/pom.xml.versionsBackup | 91 +
.../opensoc/indexing/AbstractIndexingBolt.java | 104 +
.../opensoc/indexing/TelemetryIndexingBolt.java | 241 +
.../indexing/adapters/AbstractIndexAdapter.java | 25 +
.../indexing/adapters/ESBaseBulkAdapter.java | 144 +
.../adapters/ESBulkRotatingAdapter.java | 152 +
.../adapters/ESTimedRotatingAdapter.java | 163 +
.../opensoc/indexing/adapters/SolrAdapter.java | 5 +
.../OpenSOC-MessageParsers/pom.xml | 90 +
.../pom.xml.versionsBackup | 74 +
.../com/opensoc/filters/BroMessageFilter.java | 44 +
.../opensoc/filters/GenericMessageFilter.java | 19 +
.../com/opensoc/parsing/AbstractParserBolt.java | 163 +
.../com/opensoc/parsing/PcapParserBolt.java | 256 +
.../opensoc/parsing/TelemetryParserBolt.java | 217 +
.../opensoc/parsing/parsers/AbstractParser.java | 42 +
.../opensoc/parsing/parsers/BasicBroParser.java | 114 +
.../opensoc/parsing/parsers/BasicIseParser.java | 82 +
.../parsing/parsers/BasicLancopeParser.java | 72 +
.../parsing/parsers/BasicSourcefireParser.java | 91 +
.../parsing/parsers/GrokSourcefireParser.java | 74 +
.../opensoc/parsing/parsers/JSONCleaner.java | 78 +
.../com/opensoc/parsing/parsers/PcapParser.java | 185 +
.../src/main/resources/patters/sourcefire | 30 +
.../parsing/test/BasicIseParserTest.java | 210 +
.../parsing/test/BasicLancopeParserTest.java | 156 +
.../parsing/test/BasicSourcefireParserTest.java | 107 +
.../com/opensoc/parsing/test/BroParserTest.java | 122 +
.../src/test/resources/IseSample.log | 308 +
.../test/resources/TestSchemas/IseSchema.json | 21 +
.../resources/TestSchemas/LancopeSchema.json | 22 +
.../test/resources/TestSchemas/PcapSchema.json | 22 +
.../resources/TestSchemas/SourcefireSchema.json | 22 +
.../OpenSOC-PCAP_Reconstruction/hbase/.pmd | 1262 +
.../hbase/README.txt | 16 +
.../hbase/dependency-reduced-pom.xml | 230 +
.../OpenSOC-PCAP_Reconstruction/hbase/pom.xml | 137 +
.../hbase/client/CellTimestampComparator.java | 23 +
.../opensoc/hbase/client/ConfigurationUtil.java | 267 +
.../hbase/client/HBaseConfigConstants.java | 40 +
.../hbase/client/HBaseConfigurationUtil.java | 165 +
.../cisco/opensoc/hbase/client/IPcapGetter.java | 88 +
.../opensoc/hbase/client/IPcapReceiver.java | 109 +
.../opensoc/hbase/client/IPcapScanner.java | 49 +
.../hbase/client/PcapGetterHBaseImpl.java | 790 +
.../cisco/opensoc/hbase/client/PcapHelper.java | 205 +
.../opensoc/hbase/client/PcapReceiverImpl.java | 212 +
.../hbase/client/PcapScannerHBaseImpl.java | 302 +
.../opensoc/hbase/client/PcapsResponse.java | 151 +
.../opensoc/hbase/client/RestTestingUtil.java | 238 +
.../main/resources/config-definition-hbase.xml | 34 +
.../resources/hbase-config-default.properties | 40 +
.../hbase/src/main/resources/log4j.properties | 21 +
.../client/CellTimestampComparatorTest.java | 92 +
.../hbase/client/ConfigurationUtilTest.java | 50 +
.../client/HBaseConfigurationUtilTest.java | 52 +
.../hbase/client/HBaseIntegrationTest.java | 74 +
.../hbase/client/PcapGetterHBaseImplTest.java | 536 +
.../opensoc/hbase/client/PcapHelperTest.java | 321 +
.../hbase/client/PcapReceiverImplTest.java | 232 +
.../hbase/client/PcapScannerHBaseImplTest.java | 234 +
.../src/test/resources/hbase-config.properties | 40 +
.../src/test/resources/test-tcp-packet.pcap | Bin 0 -> 144 bytes
.../OpenSOC-PCAP_Reconstruction/service/.pmd | 1190 +
.../service/README.txt | 10 +
.../OpenSOC-PCAP_Reconstruction/service/pom.xml | 52 +
.../service/src/main/resources/log4j.properties | 10 +
.../src/main/webapp/META-INF/MANIFEST.MF | 3 +
.../src/main/webapp/WEB-INF/ipcap-config.xml | 7 +
.../service/src/main/webapp/WEB-INF/web.xml | 17 +
.../WhoisEnrichment/Whois_CSV_to_JSON.py | 208 +
.../OpenSOC-Topologies/pom.xml.versionsBackup | 190 +
.../com/opensoc/test/bolts/PrintingBolt.java | 48 +
.../opensoc/test/filereaders/FileReader.java | 52 +
.../test/spouts/GenericInternalTestSpout.java | 114 +
.../opensoc/test/spouts/PcapSimulatorSpout.java | 153 +
.../src/main/java/com/opensoc/topology/Bro.java | 36 +
.../src/main/java/com/opensoc/topology/Ise.java | 35 +
.../main/java/com/opensoc/topology/Lancope.java | 35 +
.../main/java/com/opensoc/topology/Pcap.java | 35 +
.../java/com/opensoc/topology/Sourcefire.java | 40 +
.../com/opensoc/topology/runner/BroRunner.java | 78 +
.../com/opensoc/topology/runner/ISERunner.java | 78 +
.../opensoc/topology/runner/LancopeRunner.java | 81 +
.../com/opensoc/topology/runner/PcapRunner.java | 70 +
.../topology/runner/SourcefireRunner.java | 81 +
.../opensoc/topology/runner/TopologyRunner.java | 931 +
.../etc/env/environment_common.conf | 4 +
.../OpenSOC_Configs/etc/env/es_connection.conf | 3 +
.../etc/env/hdfs_connection.conf | 2 +
.../etc/env/mysql_connection.conf | 4 +
.../OpenSOC_Configs/etc/hbase-site.xml | 127 +
.../etc/whitelists/known_hosts.conf | 3 +
.../OpenSOC_Configs/topologies/bro/alerts.xml | 24 +
.../topologies/bro/features_enabled.conf | 105 +
.../OpenSOC_Configs/topologies/bro/metrics.conf | 26 +
.../topologies/bro/topology.conf | 88 +
.../topologies/bro/topology_identifier.conf | 4 +
.../topologies/environment_identifier.conf | 5 +
.../OpenSOC_Configs/topologies/ise/alerts.xml | 11 +
.../topologies/ise/features_enabled.conf | 105 +
.../OpenSOC_Configs/topologies/ise/metrics.conf | 26 +
.../topologies/ise/topology.conf | 86 +
.../topologies/ise/topology_identifier.conf | 4 +
.../topologies/lancope/alerts.xml | 11 +
.../topologies/lancope/features_enabled.conf | 105 +
.../topologies/lancope/metrics.conf | 26 +
.../topologies/lancope/topology.conf | 86 +
.../topologies/lancope/topology_identifier.conf | 4 +
.../topologies/pcap/features_enabled.conf | 110 +
.../topologies/pcap/metrics.conf | 26 +
.../topologies/pcap/topology.conf | 112 +
.../topologies/pcap/topology_identifier.conf | 4 +
.../topologies/sourcefire/alerts.xml | 11 +
.../topologies/sourcefire/features_enabled.conf | 105 +
.../topologies/sourcefire/metrics.conf | 26 +
.../topologies/sourcefire/topology.conf | 86 +
.../sourcefire/topology_identifier.conf | 4 +
.../main/resources/SampleInput/BroExampleOutput | 23411 +++++++++++++++++
.../main/resources/SampleInput/ISESampleOutput | 308 +
.../resources/SampleInput/LancopeExampleOutput | 40 +
.../resources/SampleInput/PCAPExampleOutput | 0
.../SampleInput/SourcefireExampleOutput | 2 +
.../resources/TopologyConfigs_old/lancope.conf | 90 +
opensoc-streaming/pom.xml | 113 +
opensoc-streaming/pom.xml.versionsBackup | 104 +
opensoc-streaming/readme.md | 137 +
opensoc-ui/.gitignore | 39 +
opensoc-ui/.jshintignore | 3 +
opensoc-ui/.jshintrc | 66 +
opensoc-ui/.nodemonignore | 2 +
opensoc-ui/.travis.yml | 19 +
opensoc-ui/Gruntfile.js | 29 +
opensoc-ui/LICENSE | 201 +
opensoc-ui/Makefile | 29 +
opensoc-ui/README.md | 135 +
opensoc-ui/Vagrantfile | 129 +
opensoc-ui/bower.json | 24 +
opensoc-ui/doc/README.md | 4 +
opensoc-ui/examples/pcap-panel/pcap-parse.html | 140 +
opensoc-ui/examples/server/nginx.conf | 42 +
opensoc-ui/index.js | 1 +
opensoc-ui/lib/modules/es-proxy.js | 20 +
opensoc-ui/lib/modules/login.js | 32 +
opensoc-ui/lib/modules/pcap.js | 95 +
opensoc-ui/lib/opensoc-ui.js | 102 +
opensoc-ui/lib/public/app/app.js | 158 +
.../lib/public/app/components/extend-jquery.js | 47 +
opensoc-ui/lib/public/app/components/kbn.js | 632 +
.../public/app/components/lodash.extended.js | 34 +
.../lib/public/app/components/require.config.js | 98 +
.../lib/public/app/components/settings.js | 28 +
opensoc-ui/lib/public/app/controllers/all.js | 6 +
opensoc-ui/lib/public/app/controllers/dash.js | 111 +
.../lib/public/app/controllers/dashLoader.js | 128 +
.../lib/public/app/controllers/pulldown.js | 45 +
opensoc-ui/lib/public/app/controllers/row.js | 112 +
.../public/app/dashboards/alerts-dashboard.json | 332 +
opensoc-ui/lib/public/app/dashboards/blank.json | 32 +
.../lib/public/app/dashboards/default.json | 535 +
.../lib/public/app/dashboards/logstash.js | 138 +
.../lib/public/app/dashboards/logstash.json | 227 +
opensoc-ui/lib/public/app/dashboards/noted.json | 161 +
opensoc-ui/lib/public/app/dashboards/pcap5.js | 262 +
.../lib/public/app/directives/addPanel.js | 35 +
opensoc-ui/lib/public/app/directives/all.js | 13 +
.../lib/public/app/directives/arrayJoin.js | 34 +
.../lib/public/app/directives/configModal.js | 52 +
.../lib/public/app/directives/confirmClick.js | 26 +
.../lib/public/app/directives/dashUpload.js | 37 +
.../lib/public/app/directives/esVersion.js | 24 +
.../lib/public/app/directives/kibanaPanel.js | 131 +
.../public/app/directives/kibanaSimplePanel.js | 77 +
opensoc-ui/lib/public/app/directives/ngBlur.js | 20 +
.../lib/public/app/directives/ngModelOnBlur.js | 25 +
opensoc-ui/lib/public/app/directives/tip.js | 20 +
opensoc-ui/lib/public/app/factories/store.js | 59 +
opensoc-ui/lib/public/app/filters/all.js | 133 +
.../lib/public/app/panels/bettermap/editor.html | 17 +
.../bettermap/leaflet/images/layers-2x.png | Bin 0 -> 2898 bytes
.../panels/bettermap/leaflet/images/layers.png | Bin 0 -> 1502 bytes
.../bettermap/leaflet/images/marker-icon-2x.png | Bin 0 -> 4033 bytes
.../bettermap/leaflet/images/marker-icon.png | Bin 0 -> 1747 bytes
.../bettermap/leaflet/images/marker-shadow.png | Bin 0 -> 797 bytes
.../app/panels/bettermap/leaflet/leaflet-src.js | 8724 ++++++
.../app/panels/bettermap/leaflet/leaflet.css | 463 +
.../app/panels/bettermap/leaflet/leaflet.ie.css | 51 +
.../app/panels/bettermap/leaflet/leaflet.js | 8 +
.../app/panels/bettermap/leaflet/plugins.css | 75 +
.../app/panels/bettermap/leaflet/plugins.js | 16 +
.../lib/public/app/panels/bettermap/module.css | 4 +
.../lib/public/app/panels/bettermap/module.html | 11 +
.../lib/public/app/panels/bettermap/module.js | 279 +
.../lib/public/app/panels/column/editor.html | 36 +
.../lib/public/app/panels/column/module.html | 16 +
.../lib/public/app/panels/column/module.js | 119 +
.../public/app/panels/column/panelgeneral.html | 11 +
.../public/app/panels/dashcontrol/editor.html | 44 +
.../public/app/panels/dashcontrol/module.html | 3 +
.../lib/public/app/panels/dashcontrol/module.js | 198 +
.../public/app/panels/derivequeries/editor.html | 1 +
.../public/app/panels/derivequeries/module.html | 3 +
.../public/app/panels/derivequeries/module.js | 53 +
.../lib/public/app/panels/fields/editor.html | 10 +
.../public/app/panels/fields/micropanel.html | 25 +
.../lib/public/app/panels/fields/module.html | 3 +
.../lib/public/app/panels/fields/module.js | 37 +
.../lib/public/app/panels/filtering/editor.html | 7 +
.../lib/public/app/panels/filtering/meta.html | 15 +
.../lib/public/app/panels/filtering/module.html | 83 +
.../lib/public/app/panels/filtering/module.js | 100 +
.../lib/public/app/panels/goal/editor.html | 30 +
.../lib/public/app/panels/goal/module.html | 15 +
opensoc-ui/lib/public/app/panels/goal/module.js | 259 +
.../lib/public/app/panels/histogram/editor.html | 51 +
.../lib/public/app/panels/histogram/interval.js | 57 +
.../lib/public/app/panels/histogram/module.html | 108 +
.../lib/public/app/panels/histogram/module.js | 826 +
.../app/panels/histogram/queriesEditor.html | 43 +
.../app/panels/histogram/styleEditor.html | 88 +
.../public/app/panels/histogram/timeSeries.js | 235 +
.../lib/public/app/panels/hits/editor.html | 29 +
.../lib/public/app/panels/hits/module.html | 44 +
opensoc-ui/lib/public/app/panels/hits/module.js | 303 +
.../lib/public/app/panels/map/editor.html | 15 +
.../app/panels/map/lib/jquery.jvectormap.min.js | 8 +
.../lib/public/app/panels/map/lib/map.europe.js | 1 +
.../lib/public/app/panels/map/lib/map.usa.js | 1 +
.../lib/public/app/panels/map/lib/map.world.js | 1 +
.../lib/public/app/panels/map/module.html | 63 +
opensoc-ui/lib/public/app/panels/map/module.js | 219 +
.../lib/public/app/panels/pcap/editor.html | 16 +
.../lib/public/app/panels/pcap/lib/showdown.js | 1454 +
.../lib/public/app/panels/pcap/module.html | 241 +
opensoc-ui/lib/public/app/panels/pcap/module.js | 376 +
.../lib/public/app/panels/pcap/packet_data.js | 4233 +++
.../lib/public/app/panels/query/editor.html | 7 +
.../public/app/panels/query/editors/lucene.html | 0
.../public/app/panels/query/editors/regex.html | 0
.../public/app/panels/query/editors/topN.html | 12 +
.../public/app/panels/query/help/lucene.html | 30 +
.../lib/public/app/panels/query/help/regex.html | 10 +
.../lib/public/app/panels/query/help/topN.html | 14 +
.../lib/public/app/panels/query/helpModal.html | 12 +
.../lib/public/app/panels/query/meta.html | 34 +
.../lib/public/app/panels/query/module.html | 30 +
.../lib/public/app/panels/query/module.js | 113 +
.../lib/public/app/panels/query/query.css | 49 +
.../public/app/panels/sparklines/editor.html | 23 +
.../public/app/panels/sparklines/interval.js | 57 +
.../public/app/panels/sparklines/module.html | 10 +
.../lib/public/app/panels/sparklines/module.js | 394 +
.../public/app/panels/sparklines/timeSeries.js | 216 +
.../lib/public/app/panels/stats/editor.html | 36 +
.../lib/public/app/panels/stats/module.html | 15 +
.../lib/public/app/panels/stats/module.js | 203 +
.../lib/public/app/panels/table/editor.html | 45 +
.../lib/public/app/panels/table/micropanel.html | 64 +
.../lib/public/app/panels/table/modal.html | 43 +
.../lib/public/app/panels/table/module.html | 184 +
.../lib/public/app/panels/table/module.js | 516 +
.../lib/public/app/panels/table/pagination.html | 32 +
.../lib/public/app/panels/terms/editor.html | 70 +
.../lib/public/app/panels/terms/module.html | 83 +
.../lib/public/app/panels/terms/module.js | 413 +
.../lib/public/app/panels/test-chart/module.js | 414 +
.../lib/public/app/panels/text/editor.html | 16 +
.../lib/public/app/panels/text/lib/showdown.js | 1454 +
.../lib/public/app/panels/text/module.html | 10 +
opensoc-ui/lib/public/app/panels/text/module.js | 99 +
.../public/app/panels/timepicker/custom.html | 78 +
.../public/app/panels/timepicker/editor.html | 16 +
.../public/app/panels/timepicker/module.html | 51 +
.../lib/public/app/panels/timepicker/module.js | 222 +
.../app/panels/timepicker/refreshctrl.html | 5 +
.../lib/public/app/panels/trends/editor.html | 20 +
.../lib/public/app/panels/trends/module.html | 17 +
.../lib/public/app/panels/trends/module.js | 258 +
.../lib/public/app/partials/dashLoader.html | 112 +
.../public/app/partials/dashLoaderShare.html | 11 +
.../lib/public/app/partials/dashboard.html | 86 +
.../lib/public/app/partials/dasheditor.html | 187 +
.../lib/public/app/partials/inspector.html | 15 +
opensoc-ui/lib/public/app/partials/load.html | 4 +
opensoc-ui/lib/public/app/partials/modal.html | 12 +
.../lib/public/app/partials/paneladd.html | 6 +
.../lib/public/app/partials/paneleditor.html | 24 +
.../lib/public/app/partials/panelgeneral.html | 24 +
.../lib/public/app/partials/querySelect.html | 28 +
.../lib/public/app/partials/roweditor.html | 67 +
opensoc-ui/lib/public/app/services/alertSrv.js | 49 +
opensoc-ui/lib/public/app/services/all.js | 12 +
opensoc-ui/lib/public/app/services/dashboard.js | 529 +
opensoc-ui/lib/public/app/services/esVersion.js | 195 +
opensoc-ui/lib/public/app/services/fields.js | 111 +
opensoc-ui/lib/public/app/services/filterSrv.js | 249 +
opensoc-ui/lib/public/app/services/kbnIndex.js | 115 +
opensoc-ui/lib/public/app/services/panelMove.js | 68 +
opensoc-ui/lib/public/app/services/querySrv.js | 284 +
opensoc-ui/lib/public/app/services/timer.js | 34 +
opensoc-ui/lib/public/config.js | 82 +
opensoc-ui/lib/public/css/animate.min.css | 3270 +++
.../lib/public/css/bootstrap-responsive.min.css | 9 +
.../lib/public/css/bootstrap.dark.min.css | 9 +
.../lib/public/css/bootstrap.light.min.css | 9 +
opensoc-ui/lib/public/css/font-awesome.min.css | 403 +
opensoc-ui/lib/public/css/normalize.min.css | 1 +
opensoc-ui/lib/public/css/opensoc.css | 9 +
opensoc-ui/lib/public/css/timepicker.css | 423 +
opensoc-ui/lib/public/favicon.ico | Bin 0 -> 1150 bytes
opensoc-ui/lib/public/font/FontAwesome.otf | Bin 0 -> 61896 bytes
.../lib/public/font/fontawesome-webfont.eot | Bin 0 -> 37405 bytes
.../lib/public/font/fontawesome-webfont.svg | 399 +
.../lib/public/font/fontawesome-webfont.ttf | Bin 0 -> 79076 bytes
.../lib/public/font/fontawesome-webfont.woff | Bin 0 -> 43572 bytes
opensoc-ui/lib/public/img/annotation-icon.png | Bin 0 -> 4687 bytes
opensoc-ui/lib/public/img/cubes.png | Bin 0 -> 22505 bytes
.../public/img/glyphicons-halflings-white.png | Bin 0 -> 8777 bytes
.../lib/public/img/glyphicons-halflings.png | Bin 0 -> 12799 bytes
opensoc-ui/lib/public/img/kibana.png | Bin 0 -> 15438 bytes
opensoc-ui/lib/public/img/light.png | Bin 0 -> 14080 bytes
opensoc-ui/lib/public/img/load.gif | Bin 0 -> 723 bytes
opensoc-ui/lib/public/img/load_big.gif | Bin 0 -> 2892 bytes
opensoc-ui/lib/public/img/small.png | Bin 0 -> 480 bytes
opensoc-ui/lib/public/vendor/LICENSE.json | 90 +
.../public/vendor/angular/angular-cookies.js | 185 +
.../public/vendor/angular/angular-dragdrop.js | 333 +
.../public/vendor/angular/angular-sanitize.js | 558 +
.../lib/public/vendor/angular/angular-strap.js | 878 +
opensoc-ui/lib/public/vendor/angular/angular.js | 16893 ++++++++++++
.../lib/public/vendor/angular/bindonce.js | 269 +
.../lib/public/vendor/angular/datepicker.js | 1046 +
.../lib/public/vendor/angular/timepicker.js | 888 +
opensoc-ui/lib/public/vendor/blob.js | 178 +
.../lib/public/vendor/bootstrap/bootstrap.js | 2327 ++
.../public/vendor/bootstrap/less/accordion.less | 34 +
.../public/vendor/bootstrap/less/alerts.less | 79 +
.../bootstrap/less/bak/bootswatch.dark.less | 555 +
.../bootstrap/less/bak/variables.dark.less | 304 +
.../vendor/bootstrap/less/bootstrap.dark.less | 4 +
.../public/vendor/bootstrap/less/bootstrap.less | 63 +
.../vendor/bootstrap/less/bootstrap.light.less | 4 +
.../vendor/bootstrap/less/bootswatch.dark.less | 577 +
.../vendor/bootstrap/less/bootswatch.light.less | 592 +
.../vendor/bootstrap/less/breadcrumbs.less | 24 +
.../vendor/bootstrap/less/button-groups.less | 229 +
.../public/vendor/bootstrap/less/buttons.less | 228 +
.../public/vendor/bootstrap/less/carousel.less | 158 +
.../lib/public/vendor/bootstrap/less/close.less | 32 +
.../lib/public/vendor/bootstrap/less/code.less | 61 +
.../bootstrap/less/component-animations.less | 22 +
.../public/vendor/bootstrap/less/dropdowns.less | 248 +
.../lib/public/vendor/bootstrap/less/forms.less | 690 +
.../lib/public/vendor/bootstrap/less/grid.less | 21 +
.../public/vendor/bootstrap/less/hero-unit.less | 25 +
.../vendor/bootstrap/less/labels-badges.less | 84 +
.../public/vendor/bootstrap/less/layouts.less | 16 +
.../lib/public/vendor/bootstrap/less/media.less | 55 +
.../public/vendor/bootstrap/less/mixins.less | 702 +
.../public/vendor/bootstrap/less/modals.less | 87 +
.../public/vendor/bootstrap/less/navbar.less | 497 +
.../lib/public/vendor/bootstrap/less/navs.less | 409 +
.../public/vendor/bootstrap/less/overrides.less | 601 +
.../lib/public/vendor/bootstrap/less/pager.less | 43 +
.../vendor/bootstrap/less/pagination.less | 123 +
.../public/vendor/bootstrap/less/popovers.less | 133 +
.../vendor/bootstrap/less/progress-bars.less | 122 +
.../lib/public/vendor/bootstrap/less/reset.less | 216 +
.../bootstrap/less/responsive-1200px-min.less | 28 +
.../bootstrap/less/responsive-767px-max.less | 193 +
.../bootstrap/less/responsive-768px-979px.less | 19 +
.../bootstrap/less/responsive-navbar.less | 189 +
.../bootstrap/less/responsive-utilities.less | 59 +
.../vendor/bootstrap/less/responsive.less | 48 +
.../vendor/bootstrap/less/scaffolding.less | 53 +
.../public/vendor/bootstrap/less/sprites.less | 197 +
.../public/vendor/bootstrap/less/tables.less | 244 +
.../vendor/bootstrap/less/tests/buttons.html | 139 +
.../vendor/bootstrap/less/tests/css-tests.css | 150 +
.../vendor/bootstrap/less/tests/css-tests.html | 1399 +
.../bootstrap/less/tests/forms-responsive.html | 71 +
.../vendor/bootstrap/less/tests/forms.html | 179 +
.../bootstrap/less/tests/navbar-fixed-top.html | 104 +
.../bootstrap/less/tests/navbar-static-top.html | 107 +
.../vendor/bootstrap/less/tests/navbar.html | 107 +
.../vendor/bootstrap/less/thumbnails.less | 53 +
.../public/vendor/bootstrap/less/tooltip.less | 70 +
.../lib/public/vendor/bootstrap/less/type.less | 247 +
.../public/vendor/bootstrap/less/utilities.less | 30 +
.../vendor/bootstrap/less/variables.dark.less | 303 +
.../public/vendor/bootstrap/less/variables.less | 301 +
.../vendor/bootstrap/less/variables.light.less | 304 +
.../lib/public/vendor/bootstrap/less/wells.less | 29 +
opensoc-ui/lib/public/vendor/chromath.js | 34 +
.../vendor/elasticjs/elastic-angular-client.js | 94 +
.../lib/public/vendor/elasticjs/elastic.js | 22268 ++++++++++++++++
opensoc-ui/lib/public/vendor/filesaver.js | 216 +
.../lib/public/vendor/jquery/jquery-1.8.0.js | 2 +
.../public/vendor/jquery/jquery-ui-1.10.3.js | 5253 ++++
.../public/vendor/jquery/jquery.flot.byte.js | 107 +
.../public/vendor/jquery/jquery.flot.events.js | 635 +
.../lib/public/vendor/jquery/jquery.flot.js | 3137 +++
.../lib/public/vendor/jquery/jquery.flot.pie.js | 817 +
.../vendor/jquery/jquery.flot.selection.js | 360 +
.../public/vendor/jquery/jquery.flot.stack.js | 188 +
.../vendor/jquery/jquery.flot.stackpercent.js | 126 +
.../public/vendor/jquery/jquery.flot.time.js | 431 +
opensoc-ui/lib/public/vendor/lodash.js | 6785 +++++
opensoc-ui/lib/public/vendor/modernizr-2.6.1.js | 4 +
opensoc-ui/lib/public/vendor/moment.js | 1662 ++
opensoc-ui/lib/public/vendor/numeral.js | 565 +
opensoc-ui/lib/public/vendor/peg.js | 5040 ++++
.../lib/public/vendor/require/css-build.js | 78 +
opensoc-ui/lib/public/vendor/require/css.js | 23 +
opensoc-ui/lib/public/vendor/require/require.js | 2024 ++
opensoc-ui/lib/public/vendor/require/text.js | 386 +
opensoc-ui/lib/public/vendor/require/tmpl.js | 17 +
opensoc-ui/lib/public/vendor/timezone.js | 993 +
opensoc-ui/lib/views/alerts.jade | 62 +
opensoc-ui/lib/views/index.jade | 43 +
opensoc-ui/lib/views/login.jade | 43 +
opensoc-ui/package.json | 62 +
opensoc-ui/script/es_fetch | 63 +
opensoc-ui/script/es_gen.js | 174 +
opensoc-ui/script/es_seed | 69 +
opensoc-ui/script/generate_alert | 38 +
opensoc-ui/script/ldap_seed | 5 +
opensoc-ui/script/migrate | 3 +
opensoc-ui/script/parse | 48 +
opensoc-ui/script/provision | 63 +
opensoc-ui/seed/ldap/config.ldif | 16 +
opensoc-ui/seed/ldap/content.ldif | 45 +
opensoc-ui/seed/ldap/logging.ldif | 6 +
opensoc-ui/seed/ldap/memberof_add.ldif | 5 +
opensoc-ui/seed/ldap/memberof_config.ldif | 11 +
opensoc-ui/seed/slapd.seed | 14 +
opensoc-ui/test/opensoc-ui-test.js | 15 +
opensoc-ui/test/session-test.js | 63 +
563 files changed, 175878 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..5c304d1
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,201 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..3fc347a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,61 @@
+# OpenSOC
+
+OpenSOC integrates a variety of open source big data technologies in order
+to offer a centralized tool for security monitoring and analysis. OpenSOC
+provides capabilities for log aggregation, full packet capture indexing,
+storage, advanced behavioral analytics and data enrichment, while applying
+the most current threat intelligence information to security telemetry
+within a single platform.
+
+OpenSOC can be divided into 4 areas:
+
+1. **A mechanism to capture, store, and normalize any type of security
+telemetry at extremely high rates.** Because security telemetry is constantly
+being generated, it requires a method for ingesting the data at high speeds
+and pushing it to various processing units for advanced computation and analytics.
+
+2. **Real time processing and application of enrichments** such as threat
+intelligence, geolocation, and DNS information to telemetry being collected.
+The immediate application of this information to incoming telemetry provides
+the context and situational awareness, as well as the who and where
+information critical for investigation
+
+3. **Efficient information storage** based on how the information will be used:
+
+- Logs and telemetry are stored such that they can be efficiently mined and
+analyzed for concise security visibility
+- The ability to extract and reconstruct full packets helps an analyst answer
+questions such as who the true attacker was, what data was leaked, and where
+that data was sent
+- Long-term storage not only increases visibility over time, but also enables
+advanced analytics such as machine learning techniques to be used to create
+models on the information. Incoming data can then be scored against these
+stored models for advanced anomaly detection.
+
+4. **An interface that gives a security investigator a centralized view of data
+and alerts passed through the system.** OpenSOC’s interface presents alert
+summaries with threat intelligence and enrichment data specific to that alert
+on one single page. Furthermore, advanced search capabilities and full packet
+extraction tools are presented to the analyst for investigation without the
+need to pivot into additional tools.
+
+Big data is a natural fit for powerful security analytics. The OpenSOC
+framework integrates a number of elements from the Hadoop ecosystem to provide
+a scalable platform for security analytics, incorporating such functionality as
+full-packet capture, stream processing, batch processing, real-time search, and
+telemetry aggregation. With OpenSOC, our goal is to tie big data into security
+analytics and drive towards an extensible centralized platform to effectively
+enable rapid detection and rapid response for advanced security threats.
+
+# Obtaining OpenSOC
+
+This repository is a collection of submodules for convenience which is regularly
+updated to point to the latest versions. Github provides multiple ways to obtain
+OpenSOC's code:
+
+1. git clone --recursive https://github.com/OpenSOC/opensoc
+2. [Download ZIP](https://github.com/OpenSOC/opensoc/archive/master.zip)
+3. Clone or download each repository individually
+
+Option 3 is more likely to have the latest code.
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/.gitignore
----------------------------------------------------------------------
diff --git a/opensoc-streaming/.gitignore b/opensoc-streaming/.gitignore
new file mode 100644
index 0000000..6f6d811
--- /dev/null
+++ b/opensoc-streaming/.gitignore
@@ -0,0 +1,15 @@
+*.class
+target/
+copy/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+
+# Eclipse related files
+.classpath
+.project
+.settings/
+
+OpenSOC-Topologies/pom.xml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/.travis.yml
----------------------------------------------------------------------
diff --git a/opensoc-streaming/.travis.yml b/opensoc-streaming/.travis.yml
new file mode 100644
index 0000000..7c87471
--- /dev/null
+++ b/opensoc-streaming/.travis.yml
@@ -0,0 +1,14 @@
+language: java
+before_script:
+ mvn clean install -Dmode=local
+notifications:
+ email:
+ recipients:
+ - spiddapa@cisco.com
+ - jsirota@cisco.com
+ on_success: always
+ on_failure: always
+ hipchat:
+ rooms:
+ secure: grhlFGHjjEIiOUa/Wt7pyB78La9WHQCQOZEsGSjaYDAObIClBtmfP0TYEIa+Sk6auNFWdRwhxCu8xGtT+G554loR+9A5iQaCGqlJGQyygHl5PwlWu0kWRFRO75zfvLMTQ+beowM/tgGnf0MBz5adjZmnGu+L0Fet3SYcQOJixe0=
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/LICENSE
----------------------------------------------------------------------
diff --git a/opensoc-streaming/LICENSE b/opensoc-streaming/LICENSE
new file mode 100644
index 0000000..5c304d1
--- /dev/null
+++ b/opensoc-streaming/LICENSE
@@ -0,0 +1,201 @@
+Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Alerts/pom.xml
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Alerts/pom.xml b/opensoc-streaming/OpenSOC-Alerts/pom.xml
new file mode 100644
index 0000000..a2fcfe5
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Alerts/pom.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.opensoc</groupId>
+ <artifactId>OpenSOC-Streaming</artifactId>
+ <version>0.3BETA-SNAPSHOT</version>
+ </parent>
+ <artifactId>OpenSOC-Alerts</artifactId>
+ <name>OpenSOC-Alerts</name>
+ <description>Taggers for alerts</description>
+ <properties>
+ <commons.validator.version>1.4.0</commons.validator.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>com.opensoc</groupId>
+ <artifactId>OpenSOC-Common</artifactId>
+ <version>${parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>${global_json_simple_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${global_storm_version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.9.2</artifactId>
+ <version>${global_kafka_version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${global_metrics_version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-validator</groupId>
+ <artifactId>commons-validator</artifactId>
+ <version>${commons.validator.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Alerts/pom.xml.versionsBackup
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Alerts/pom.xml.versionsBackup b/opensoc-streaming/OpenSOC-Alerts/pom.xml.versionsBackup
new file mode 100644
index 0000000..3530449
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Alerts/pom.xml.versionsBackup
@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?><!-- Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>com.opensoc</groupId>
+ <artifactId>OpenSOC-Streaming</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>OpenSOC-Alerts</artifactId>
+ <name>OpenSOC-Alerts</name>
+ <description>Taggers for alerts</description>
+ <properties>
+ <json.simple.version>1.1.1</json.simple.version>
+
+ <storm.version>0.9.2-incubating</storm.version>
+ <kafka.version>0.8.0</kafka.version>
+ <metrics.version>3.0.2</metrics.version>
+ <commons.config.version>1.10</commons.config.version>
+ <opensoc.common.version>0.0.1-SNAPSHOT</opensoc.common.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>com.opensoc</groupId>
+ <artifactId>OpenSOC-Common</artifactId>
+ <version>${opensoc.common.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>${json.simple.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.storm</groupId>
+ <artifactId>storm-core</artifactId>
+ <version>${storm.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.9.2</artifactId>
+ <version>0.8.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.sun.jmx</groupId>
+ <artifactId>jmxri</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jdmk</groupId>
+ <artifactId>jmxtools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-graphite</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>${commons.config.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-validator</groupId>
+ <artifactId>commons-validator</artifactId>
+ <version>1.4.0</version>
+ </dependency>
+
+ </dependencies>
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/AbstractAlertBolt.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/AbstractAlertBolt.java b/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/AbstractAlertBolt.java
new file mode 100644
index 0000000..8dc4c9d
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/AbstractAlertBolt.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.opensoc.alerts;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.topology.base.BaseRichBolt;
+import backtype.storm.tuple.Fields;
+
+import com.codahale.metrics.Counter;
+import com.google.common.cache.Cache;
+import com.opensoc.alerts.interfaces.AlertsAdapter;
+import com.opensoc.metrics.MetricReporter;
+
+@SuppressWarnings("rawtypes")
+public abstract class AbstractAlertBolt extends BaseRichBolt {
+ /**
+ *
+ */
+ private static final long serialVersionUID = -6710596708304282838L;
+
+ transient Cache<String, String> cache;
+
+ protected static final Logger LOG = LoggerFactory
+ .getLogger(AbstractAlertBolt.class);
+
+ protected OutputCollector _collector;
+ protected AlertsAdapter _adapter;
+
+ protected String OutputFieldName;
+ protected JSONObject _identifier;
+ protected MetricReporter _reporter;
+
+ protected int _MAX_CACHE_SIZE = -1;
+ protected int _MAX_TIME_RETAIN = -1;
+
+ protected Counter ackCounter, emitCounter, failCounter;
+
+ protected void registerCounters() {
+
+ String ackString = _adapter.getClass().getSimpleName() + ".ack";
+
+ String emitString = _adapter.getClass().getSimpleName() + ".emit";
+
+ String failString = _adapter.getClass().getSimpleName() + ".fail";
+
+ ackCounter = _reporter.registerCounter(ackString);
+ emitCounter = _reporter.registerCounter(emitString);
+ failCounter = _reporter.registerCounter(failString);
+
+ }
+
+ public final void prepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) {
+ _collector = collector;
+
+ if (this._adapter == null)
+ throw new IllegalStateException("Alerts adapter must be specified");
+ if (this._identifier == null)
+ throw new IllegalStateException("Identifier must be specified");
+
+ if (this._MAX_CACHE_SIZE == -1)
+ throw new IllegalStateException("MAX_CACHE_SIZE must be specified");
+ if (this._MAX_TIME_RETAIN == -1)
+ throw new IllegalStateException("MAX_TIME_RETAIN must be specified");
+
+ try {
+ doPrepare(conf, topologyContext, collector);
+ } catch (IOException e) {
+ LOG.error("Counld not initialize...");
+ e.printStackTrace();
+ }
+
+ boolean success = _adapter.initialize();
+ try {
+ if (!success)
+
+ throw new Exception("Could not initialize adapter");
+ } catch (Exception e) {
+
+ e.printStackTrace();
+ }
+ }
+
+ public void declareOutputFields(OutputFieldsDeclarer declearer) {
+ declearer.declareStream("message", new Fields("key", "message"));
+ declearer.declareStream("alert", new Fields( "message"));
+ declearer.declareStream("error", new Fields("message"));
+ }
+
+ abstract void doPrepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) throws IOException;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/TelemetryAlertsBolt.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/TelemetryAlertsBolt.java b/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/TelemetryAlertsBolt.java
new file mode 100644
index 0000000..36bd0fc
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/TelemetryAlertsBolt.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.opensoc.alerts;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.configuration.Configuration;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+
+import backtype.storm.task.OutputCollector;
+import backtype.storm.task.TopologyContext;
+import backtype.storm.tuple.Tuple;
+import backtype.storm.tuple.Values;
+
+import com.esotericsoftware.minlog.Log;
+import com.google.common.cache.CacheBuilder;
+import com.opensoc.alerts.interfaces.AlertsAdapter;
+import com.opensoc.json.serialization.JSONEncoderHelper;
+import com.opensoc.metrics.MetricReporter;
+import com.opensoc.topologyhelpers.ErrorGenerator;
+
+@SuppressWarnings("rawtypes")
+public class TelemetryAlertsBolt extends AbstractAlertBolt {
+
+ /**
+ * Use an adapter to tag existing telemetry messages with alerts. The list
+ * of available tagger adapters is located under
+ * com.opensoc.tagging.adapters. At the time of the release the following
+ * adapters are available:
+ *
+ * <p>
+ * <ul>
+ * <li>RegexTagger = read a list or regular expressions and tag a message if
+ * they exist in a message
+ * <li>StaticAllTagger = tag each message with a static alert
+ * <ul>
+ * <p>
+ */
+ private static final long serialVersionUID = -2647123143398352020L;
+ private Properties metricProperties;
+ private JSONObject metricConfiguration;
+
+ // private AlertsCache suppressed_alerts;
+
+ /**
+ *
+ * @param tagger
+ * - tagger adapter for generating alert tags
+ * @return instance of bolt
+ */
+ public TelemetryAlertsBolt withAlertsAdapter(AlertsAdapter tagger) {
+ _adapter = tagger;
+ return this;
+ }
+
+ /**
+ *
+ * @param OutputFieldName
+ * - output name of the tuple coming out of this bolt
+ * @return - instance of this bolt
+ */
+ public TelemetryAlertsBolt withOutputFieldName(String OutputFieldName) {
+ this.OutputFieldName = OutputFieldName;
+ return this;
+ }
+
+ /**
+ *
+ * @param metricProperties
+ * - metric output to graphite
+ * @return - instance of this bolt
+ */
+ public TelemetryAlertsBolt withMetricProperties(Properties metricProperties) {
+ this.metricProperties = metricProperties;
+ return this;
+ }
+
+ /**
+ *
+ * @param identifier
+ * - the identifier tag for tagging telemetry messages with
+ * alerts out of this bolt
+ * @return - instance of this bolt
+ */
+
+ public TelemetryAlertsBolt withIdentifier(JSONObject identifier) {
+ this._identifier = identifier;
+ return this;
+ }
+
+ /**
+ * @param config
+ * A class for generating custom metrics into graphite
+ * @return Instance of this class
+ */
+
+ public TelemetryAlertsBolt withMetricConfiguration(Configuration config) {
+ this.metricConfiguration = JSONEncoderHelper.getJSON(config
+ .subset("com.opensoc.metrics"));
+ return this;
+ }
+
+ /**
+ * @param MAX_CACHE_SIZE
+ * Maximum size of cache before flushing
+ * @return Instance of this class
+ */
+
+ public TelemetryAlertsBolt withMaxCacheSize(int MAX_CACHE_SIZE) {
+ _MAX_CACHE_SIZE = MAX_CACHE_SIZE;
+ return this;
+ }
+
+ /**
+ * @param MAX_TIME_RETAIN
+ * Maximum time to retain cached entry before expiring
+ * @return Instance of this class
+ */
+
+ public TelemetryAlertsBolt withMaxTimeRetain(int MAX_TIME_RETAIN) {
+ _MAX_TIME_RETAIN = MAX_TIME_RETAIN;
+ return this;
+ }
+
+ @Override
+ void doPrepare(Map conf, TopologyContext topologyContext,
+ OutputCollector collector) throws IOException {
+
+ cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE)
+ .expireAfterWrite(_MAX_TIME_RETAIN, TimeUnit.MINUTES).build();
+
+ LOG.info("[OpenSOC] Preparing TelemetryAlert Bolt...");
+
+ try {
+ _reporter = new MetricReporter();
+ _reporter.initialize(metricProperties, TelemetryAlertsBolt.class);
+ LOG.info("[OpenSOC] Initialized metrics");
+ } catch (Exception e) {
+ LOG.info("[OpenSOC] Could not initialize metrics");
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void execute(Tuple tuple) {
+
+ LOG.trace("[OpenSOC] Starting to process message for alerts");
+ JSONObject original_message = null;
+ String key = null;
+
+ try {
+
+ key = tuple.getStringByField("key");
+ original_message = (JSONObject) tuple.getValueByField("message");
+
+ if (original_message == null || original_message.isEmpty())
+ throw new Exception("Could not parse message from byte stream");
+
+ if(key == null)
+ throw new Exception("Key is not valid");
+
+ LOG.trace("[OpenSOC] Received tuple: " + original_message);
+
+ JSONObject alerts_tag = new JSONObject();
+ Map<String, JSONObject> alerts_list = _adapter
+ .alert(original_message);
+ JSONArray uuid_list = new JSONArray();
+
+ if (alerts_list == null || alerts_list.isEmpty()) {
+ LOG.trace("[OpenSOC] No alerts detected in: "
+ + original_message);
+ _collector.ack(tuple);
+ _collector.emit(new Values(original_message));
+ } else {
+ for (String alert : alerts_list.keySet()) {
+ uuid_list.add(alert);
+
+ LOG.trace("[OpenSOC] Checking alerts cache: " + alert);
+
+ if (cache.getIfPresent(alert) == null) {
+ LOG.trace("[OpenSOC]: Alert not found in cache: " + alert);
+
+ JSONObject global_alert = new JSONObject();
+ global_alert.putAll(_identifier);
+ global_alert.put("triggered", alerts_list.get(alert));
+ global_alert.put("timestamp", System.currentTimeMillis());
+ _collector.emit("alert", new Values(global_alert));
+
+ cache.put(alert, "");
+
+ } else
+ LOG.trace("Alert located in cache: " + alert);
+
+ LOG.debug("[OpenSOC] Alerts are: " + alerts_list);
+
+ if (original_message.containsKey("alerts")) {
+ JSONArray already_triggered = (JSONArray) original_message
+ .get("alerts");
+
+ uuid_list.addAll(already_triggered);
+ LOG.trace("[OpenSOC] Messages already had alerts...tagging more");
+ }
+
+ original_message.put("alerts", uuid_list);
+
+ LOG.debug("[OpenSOC] Detected alerts: " + alerts_tag);
+
+ _collector.ack(tuple);
+ _collector.emit("message", new Values(key, original_message));
+
+ }
+
+ /*
+ * if (metricConfiguration != null) { emitCounter.inc();
+ * ackCounter.inc(); }
+ */
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOG.error("Failed to tag message :" + original_message);
+ e.printStackTrace();
+ _collector.fail(tuple);
+
+ /*
+ * if (metricConfiguration != null) { failCounter.inc(); }
+ */
+
+ String error_as_string = org.apache.commons.lang.exception.ExceptionUtils
+ .getStackTrace(e);
+
+ JSONObject error = ErrorGenerator.generateErrorMessage(
+ "Alerts problem: " + original_message, error_as_string);
+ _collector.emit("error", new Values(error));
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/05e188ba/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/adapters/AbstractAlertAdapter.java
----------------------------------------------------------------------
diff --git a/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/adapters/AbstractAlertAdapter.java b/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/adapters/AbstractAlertAdapter.java
new file mode 100644
index 0000000..35595a0
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Alerts/src/main/java/com/opensoc/alerts/adapters/AbstractAlertAdapter.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.opensoc.alerts.adapters;
+
+import java.io.Serializable;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.opensoc.alerts.interfaces.AlertsAdapter;
+
+@SuppressWarnings("serial")
+public abstract class AbstractAlertAdapter implements AlertsAdapter, Serializable{
+
+ protected static final Logger _LOG = LoggerFactory
+ .getLogger(AbstractAlertAdapter.class);
+
+
+ protected Cache<String, String> cache;
+
+ protected String generateAlertId(String source_ip, String dst_ip,
+ int alert_type) {
+
+ String key = makeKey(source_ip, dst_ip, alert_type);
+
+ if (cache.getIfPresent(key) != null)
+ return cache.getIfPresent(key);
+
+ String new_UUID = System.currentTimeMillis() + "-" + UUID.randomUUID();
+
+ cache.put(key, new_UUID);
+ key = makeKey(dst_ip, source_ip, alert_type);
+ cache.put(key, new_UUID);
+
+ return new_UUID;
+
+ }
+
+ private String makeKey(String ip1, String ip2, int alert_type) {
+ return (ip1 + "-" + ip2 + "-" + alert_type);
+ }
+
+ private void generateCache(int _MAX_CACHE_SIZE, int _MAX_TIME_RETAIN)
+ {
+ cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE)
+ .expireAfterWrite(_MAX_TIME_RETAIN, TimeUnit.MINUTES).build();
+ }
+}