You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/04/05 21:42:11 UTC
[15/15] incubator-metron git commit: METRON 86: Adding Solr indexing
support (merrimanr via cestella) closes apache/incubator-metron#67
METRON 86: Adding Solr indexing support (merrimanr via cestella) closes apache/incubator-metron#67
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/e59b1a31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/e59b1a31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/e59b1a31
Branch: refs/heads/master
Commit: e59b1a31da16a3bff5f00b0947cff899f4dc1d32
Parents: edeec01
Author: merrimanr <me...@gmail.com>
Authored: Tue Apr 5 15:41:31 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Tue Apr 5 15:41:31 2016 -0400
----------------------------------------------------------------------
deployment/amazon-ec2/conf/defaults.yml | 12 +-
.../inventory/metron_example/group_vars/all | 11 +-
.../inventory/multinode-vagrant/group_vars/all | 32 +-
.../inventory/singlenode-vagrant/group_vars/all | 6 +-
deployment/playbooks/metron_install.yml | 15 +-
.../roles/metron_streaming/defaults/main.yml | 6 +-
.../files/config/sensors/bro.json | 14 +
.../files/config/sensors/pcap.json | 14 +
.../files/config/sensors/snort.json | 14 +
.../files/config/sensors/yaf.json | 14 +
.../files/source/bro-config.json | 14 -
.../files/source/pcap-config.json | 14 -
.../files/source/snort-config.json | 14 -
.../files/source/yaf-config.json | 14 -
.../roles/metron_streaming/handlers/main.yml | 4 +-
.../roles/metron_streaming/tasks/main.yml | 59 +-
.../metron_streaming/tasks/metron_topology.yml | 13 +-
.../metron_streaming/tasks/source_config.yml | 29 +-
.../templates/config/elasticsearch.global.json | 6 +
.../templates/config/solr.global.json | 6 +
deployment/roles/metron_streaming/vars/main.yml | 18 +-
deployment/roles/solr/defaults/main.yml | 29 +
deployment/roles/solr/files/schema.xml | 191 +
deployment/roles/solr/meta/main.yml | 21 +
deployment/roles/solr/tasks/main.yml | 74 +
deployment/roles/solr/templates/solr.xml | 52 +
deployment/roles/solr/templates/solrconfig.xml | 583 +
metron-streaming/Metron-Common/pom.xml | 6 +
.../main/java/org/apache/metron/Constants.java | 9 +-
.../metron/bolt/BulkMessageWriterBolt.java | 38 +-
.../org/apache/metron/bolt/ConfiguredBolt.java | 70 +-
.../apache/metron/domain/Configurations.java | 92 +
.../metron/domain/SensorEnrichmentConfig.java | 62 +
.../org/apache/metron/domain/SourceConfig.java | 88 -
.../metron/helpers/topology/ErrorUtils.java | 2 +-
.../java/org/apache/metron/pcap/PcapUtils.java | 12 +
.../apache/metron/topology/TopologyUtils.java | 4 +-
.../metron/utils/ConfigurationsUtils.java | 179 +
.../java/org/apache/metron/utils/JSONUtils.java | 4 +
.../org/apache/metron/writer/HBaseWriter.java | 4 +-
.../writer/interfaces/BulkMessageWriter.java | 6 +-
.../metron/writer/interfaces/MessageWriter.java | 4 +-
.../org/apache/metron/pcap/PcapUtilsTest.java | 31 +
.../resources/config/source/bro-config.json | 14 -
.../resources/config/source/pcap-config.json | 13 -
.../resources/config/source/snort-config.json | 14 -
.../resources/config/source/yaf-config.json | 14 -
metron-streaming/Metron-Elasticsearch/pom.xml | 202 +
.../src/main/assembly/assembly.xml | 41 +
.../metron/writer/ElasticsearchWriter.java | 94 +
.../etc/env/elasticsearch.properties | 109 +
.../ElasticsearchEnrichmentIntegrationTest.java | 88 +
.../components/ElasticSearchComponent.java | 186 +
.../enrichment/bolt/EnrichmentJoinBolt.java | 12 +-
.../enrichment/bolt/EnrichmentSplitterBolt.java | 11 +-
.../enrichment/bolt/GenericEnrichmentBolt.java | 25 +-
.../enrichment/bolt/ThreatIntelJoinBolt.java | 4 +-
.../bolt/ThreatIntelSplitterBolt.java | 4 +-
metron-streaming/Metron-Indexing/pom.xml | 5 -
.../metron/indexing/AbstractIndexingBolt.java | 110 -
.../metron/indexing/TelemetryIndexingBolt.java | 256 -
.../indexing/adapters/AbstractIndexAdapter.java | 42 -
.../indexing/adapters/ESBaseBulkAdapter.java | 165 -
.../adapters/ESBulkRotatingAdapter.java | 177 -
.../adapters/ESTimedRotatingAdapter.java | 208 -
.../metron/indexing/adapters/SolrAdapter.java | 22 -
.../metron/writer/ElasticsearchWriter.java | 101 -
.../apache/metron/writer/hdfs/HdfsWriter.java | 6 +-
.../java/org/apache/metron/bolt/ParserBolt.java | 10 +-
.../org/apache/metron/writer/KafkaWriter.java | 4 +-
metron-streaming/Metron-Solr/pom.xml | 204 +
.../Metron-Solr/src/main/assembly/assembly.xml | 41 +
.../org/apache/metron/solr/SolrConstants.java | 29 +
.../metron/writer/solr/MetronSolrClient.java | 72 +
.../apache/metron/writer/solr/SolrWriter.java | 108 +
.../Metron_Configs/etc/env/solr.properties | 109 +
.../SolrEnrichmentIntegrationTest.java | 107 +
.../integration/components/SolrComponent.java | 153 +
.../writer/solr/MetronSolrClientTest.java | 82 +
.../metron/writer/solr/SolrWriterTest.java | 139 +
.../test/resources/solr/conf/_rest_managed.json | 1 +
.../src/test/resources/solr/conf/currency.xml | 67 +
.../resources/solr/conf/lang/stopwords_en.txt | 54 +
.../src/test/resources/solr/conf/protwords.txt | 21 +
.../src/test/resources/solr/conf/schema.xml | 191 +
.../src/test/resources/solr/conf/solrconfig.xml | 583 +
.../src/test/resources/solr/conf/stopwords.txt | 14 +
.../src/test/resources/solr/conf/synonyms.txt | 29 +
.../src/test/resources/solr/solr.xml | 14 +
metron-streaming/Metron-Testing/pom.xml | 11 +-
.../metron/integration/BaseIntegrationTest.java | 48 +
.../integration/EnrichmentIntegrationTest.java | 429 +
.../metron/integration/util/TestUtils.java | 37 +
.../components/ElasticSearchComponent.java | 188 -
.../integration/util/mock/MockGeoAdapter.java | 63 +
.../util/mock/MockHBaseConnector.java | 52 +
.../util/threatintel/ThreatIntelHelper.java | 39 +
.../java/org/apache/metron/util/SampleUtil.java | 40 +
.../main/resources/sample/config/global.json | 10 +
.../resources/sample/config/sensors/bro.json | 14 +
.../resources/sample/config/sensors/pcap.json | 13 +
.../resources/sample/config/sensors/snort.json | 14 +
.../resources/sample/config/sensors/yaf.json | 14 +
.../sample/data/SampleIndexed/YafIndexed | 10 +
.../data/SampleInput/.PCAPExampleOutput.crc | Bin 0 -> 44 bytes
.../resources/sample/data/SampleInput/AsaOutput | 100 +
.../sample/data/SampleInput/BroExampleOutput | 23411 +++++++++++++++++
.../data/SampleInput/FireeyeExampleOutput | 90 +
.../sample/data/SampleInput/ISESampleOutput | 308 +
.../data/SampleInput/LancopeExampleOutput | 40 +
.../sample/data/SampleInput/PCAPExampleOutput | Bin 0 -> 4510 bytes
.../sample/data/SampleInput/PaloaltoOutput | 100 +
.../sample/data/SampleInput/SnortOutput | 3 +
.../data/SampleInput/SourcefireExampleOutput | 2 +
.../sample/data/SampleInput/YafExampleOutput | 10 +
.../sample/data/SampleParsed/SnortParsed | 3 +
.../sample/data/SampleParsed/YafExampleParsed | 10 +
metron-streaming/Metron-Topologies/pom.xml | 20 +-
.../src/main/assembly/assembly.xml | 6 +-
.../main/bash/start_elasticsearch_topology.sh | 22 +
.../src/main/bash/start_solr_topology.sh | 22 +
.../src/main/bash/start_topology.sh | 22 -
.../src/main/bash/zk_load_configs.sh | 2 +-
.../apache/metron/utils/SourceConfigUtils.java | 143 -
.../Metron_Configs/etc/env/config.properties | 3 +
.../topologies/enrichment/remote.yaml | 7 +-
.../topologies/enrichment/test.yaml | 10 +-
.../src/main/resources/SampleIndexed/YafIndexed | 10 -
.../SampleInput/.PCAPExampleOutput.crc | Bin 44 -> 0 bytes
.../src/main/resources/SampleInput/AsaOutput | 100 -
.../main/resources/SampleInput/BroExampleOutput | 23411 -----------------
.../resources/SampleInput/FireeyeExampleOutput | 90 -
.../main/resources/SampleInput/ISESampleOutput | 308 -
.../resources/SampleInput/LancopeExampleOutput | 40 -
.../resources/SampleInput/PCAPExampleOutput | Bin 4510 -> 0 bytes
.../main/resources/SampleInput/PaloaltoOutput | 100 -
.../src/main/resources/SampleInput/SnortOutput | 3 -
.../SampleInput/SourcefireExampleOutput | 2 -
.../main/resources/SampleInput/YafExampleOutput | 10 -
.../src/main/resources/SampleParsed/SnortParsed | 3 -
.../resources/SampleParsed/YafExampleParsed | 10 -
.../integration/EnrichmentIntegrationTest.java | 469 -
.../integration/ParserIntegrationTest.java | 35 +-
.../integration/PcapParserIntegrationTest.java | 58 +-
.../integration/SnortIntegrationTest.java | 24 +-
.../metron/integration/YafIntegrationTest.java | 24 +-
.../metron/integration/util/TestUtils.java | 37 -
.../integration/util/mock/MockGeoAdapter.java | 63 -
.../util/mock/MockHBaseConnector.java | 52 -
.../util/threatintel/ThreatIntelHelper.java | 40 -
metron-streaming/pom.xml | 8 +-
pom.xml | 134 +-
152 files changed, 29457 insertions(+), 26735 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/conf/defaults.yml b/deployment/amazon-ec2/conf/defaults.yml
index abe96b8..77e841d 100644
--- a/deployment/amazon-ec2/conf/defaults.yml
+++ b/deployment/amazon-ec2/conf/defaults.yml
@@ -42,11 +42,6 @@ threatintel_ip_hbase_table: malicious_ip
num_partitions: 3
retention_in_gb: 25
-# elasticsearch
-elasticsearch_transport_port: 9300
-elasticsearch_network_interface: eth0
-elasticsearch_web_port: 9200
-
# metron variables
metron_version: 0.1BETA
java_home: /usr/jdk64/jdk1.8.0_40
@@ -74,3 +69,10 @@ jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
storm_local_dir: "/data1/hadoop/storm"
kafka_log_dirs: "/data2/kafka-log"
elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
+
+#Search
+install_elasticsearch: True
+install_solr: False
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth1
+elasticsearch_web_port: 9200
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all
index 3a26769..c5d81b0 100644
--- a/deployment/inventory/metron_example/group_vars/all
+++ b/deployment/inventory/metron_example/group_vars/all
@@ -31,11 +31,6 @@ pcap_hbase_table: pcap
tracker_hbase_table: access_tracker
threatintel_ip_hbase_table: malicious_ip
-#elasticsearch
-elasticsearch_transport_port: 9300
-elasticsearch_network_interface: eth0
-elasticsearch_web_port: 9200
-
# metron variables
metron_version: 0.1BETA
java_home: /usr/jdk64/jdk1.8.0_40
@@ -70,3 +65,9 @@ pcap_replay_interface: eth1
#kafka_log_dirs: "/newdir/kafka-log"
#elasticsearch_data_dir: "/newdir1/elasticsearch"
+#Search
+install_elasticsearch: True
+install_solr: False
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth1
+elasticsearch_web_port: 9200
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/inventory/multinode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/multinode-vagrant/group_vars/all b/deployment/inventory/multinode-vagrant/group_vars/all
index fc3b56d..e486296 100644
--- a/deployment/inventory/multinode-vagrant/group_vars/all
+++ b/deployment/inventory/multinode-vagrant/group_vars/all
@@ -49,17 +49,23 @@ snort_version: "2.9.8.0-1"
snort_alert_csv_path: "/var/log/snort/alert.csv"
#data directories
-zookeeper_data_dir: "/newdir/hadoop/zookeeper"
-namenode_checkpoint_dir: "/newdir/hadoop/hdfs/namesecondary"
-namenode_name_dir: "/newdir/hadoop/hdfs/namenode"
-datanode_data_dir: "/newdir/hadoop/hdfs/data"
-journalnode_edits_dir: "/newdir/hadoop/hdfs/journalnode"
-nodemanager_local_dirs: "/newdir/hadoop/yarn/local"
-timeline_ldb_store_path: "/newdir/hadoop/yarn/timeline"
-timeline_ldb_state_path: "/newdir/hadoop/yarn/timeline"
-nodemanager_log_dirs: "/newdir/hadoop/yarn/log"
-jhs_recovery_store_ldb_path: "/newdir/hadoop/mapreduce/jhs"
-storm_local_dir: "/newdir/hadoop/storm"
-kafka_log_dirs: "/newdir/kafka-log"
-elasticsearch_data_dir: "/newdir1/elasticsearch"
+#zookeeper_data_dir: "/newdir/hadoop/zookeeper"
+#namenode_checkpoint_dir: "/newdir/hadoop/hdfs/namesecondary"
+#namenode_name_dir: "/newdir/hadoop/hdfs/namenode"
+#datanode_data_dir: "/newdir/hadoop/hdfs/data"
+#journalnode_edits_dir: "/newdir/hadoop/hdfs/journalnode"
+#nodemanager_local_dirs: "/newdir/hadoop/yarn/local"
+#timeline_ldb_store_path: "/newdir/hadoop/yarn/timeline"
+#timeline_ldb_state_path: "/newdir/hadoop/yarn/timeline"
+#nodemanager_log_dirs: "/newdir/hadoop/yarn/log"
+#jhs_recovery_store_ldb_path: "/newdir/hadoop/mapreduce/jhs"
+#storm_local_dir: "/newdir/hadoop/storm"
+#kafka_log_dirs: "/newdir/kafka-log"
+#elasticsearch_data_dir: "/newdir1/elasticsearch"
+#Search
+install_elasticsearch: True
+install_solr: False
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth1
+elasticsearch_web_port: 9200
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index c1c199b..364598f 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -76,6 +76,10 @@ pcapservice_port: 8081
#Search
install_elasticsearch: True
+install_solr: False
+solr_collection_name: Metron
+solr_number_shards: 1
+solr_replication_factor: 1
elasticsearch_transport_port: 9300
elasticsearch_network_interface: eth1
-elasticsearch_web_port: 9200
+elasticsearch_web_port: 9200
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml
index c8c181e..89db980 100644
--- a/deployment/playbooks/metron_install.yml
+++ b/deployment/playbooks/metron_install.yml
@@ -41,9 +41,16 @@
vars:
es_hosts: "{% set comma = joiner(',') %}{% for host in groups['search'] -%}{{ comma() }}{{ host }}{%- endfor %}"
roles:
- - role: elasticsearch
+ - { role: elasticsearch, when: install_elasticsearch | default(True) == True }
tags:
- - elasticsearch
+ - search
+
+- hosts: search
+ become: true
+ roles:
+ - { role: solr, when: install_solr | default(False) == True }
+ tags:
+ - search
- hosts: mysql
become: true
@@ -82,7 +89,7 @@
- hosts: web
become: true
roles:
- - role: metron_ui
- - role: metron_pcapservice
+ - { role: metron_ui, when: install_elasticsearch | default(True) == True }
+ - { role: metron_pcapservice, when: install_elasticsearch | default(True) == True }
tags:
- web
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml
index 0dd21e3..a65680e 100644
--- a/deployment/roles/metron_streaming/defaults/main.yml
+++ b/deployment/roles/metron_streaming/defaults/main.yml
@@ -15,7 +15,9 @@
# limitations under the License.
#
---
-source_config_path: "{{ metron_directory }}/config/source"
+config_path: "{{ metron_directory }}/config"
+zookeeper_config_path: "{{ config_path }}/zookeeper"
+zookeeper_global_config_path: "{{ zookeeper_config_path }}/global.json"
threat_intel_bulk_load: True
threat_intel_bin: "{{ metron_directory }}/bin/threatintel_bulk_load.sh"
threat_intel_host: "{{ groups.ambari_master[0] }}"
@@ -43,4 +45,4 @@ hdfs_config_path: "/etc/hadoop/conf"
metron_hdfs_output_dir: "/apps/metron"
metron_hdfs_rotation_policy: org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
metron_hdfs_rotation_policy_count: 1
-metron_hdfs_rotation_policy_units: DAYS
\ No newline at end of file
+metron_hdfs_rotation_policy_units: DAYS
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/files/config/sensors/bro.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/config/sensors/bro.json b/deployment/roles/metron_streaming/files/config/sensors/bro.json
new file mode 100644
index 0000000..34109b8
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/config/sensors/bro.json
@@ -0,0 +1,14 @@
+{
+ "index": "bro",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_dst_addr", "ip_src_addr"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/files/config/sensors/pcap.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/config/sensors/pcap.json b/deployment/roles/metron_streaming/files/config/sensors/pcap.json
new file mode 100644
index 0000000..4b9c639
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/config/sensors/pcap.json
@@ -0,0 +1,14 @@
+{
+ "index": "pcap",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_src_addr", "ip_dst_addr"],
+ "host": ["ip_src_addr", "ip_dst_addr"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_src_addr", "ip_dst_addr"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/files/config/sensors/snort.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/config/sensors/snort.json b/deployment/roles/metron_streaming/files/config/sensors/snort.json
new file mode 100644
index 0000000..1208637
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/config/sensors/snort.json
@@ -0,0 +1,14 @@
+{
+ "index": "snort",
+ "batchSize": 1,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_dst_addr", "ip_src_addr"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/files/config/sensors/yaf.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/config/sensors/yaf.json b/deployment/roles/metron_streaming/files/config/sensors/yaf.json
new file mode 100644
index 0000000..65de961
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/config/sensors/yaf.json
@@ -0,0 +1,14 @@
+{
+ "index": "yaf",
+ "batchSize": 5,
+ "enrichmentFieldMap":
+ {
+ "geo": ["ip_dst_addr", "ip_src_addr"],
+ "host": ["host"]
+ },
+ "threatIntelFieldMap":
+ {
+ "ip": ["ip_dst_addr", "ip_src_addr"]
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/files/source/bro-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/bro-config.json b/deployment/roles/metron_streaming/files/source/bro-config.json
deleted file mode 100644
index 34109b8..0000000
--- a/deployment/roles/metron_streaming/files/source/bro-config.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "index": "bro",
- "batchSize": 5,
- "enrichmentFieldMap":
- {
- "geo": ["ip_dst_addr", "ip_src_addr"],
- "host": ["host"]
- },
- "threatIntelFieldMap":
- {
- "ip": ["ip_dst_addr", "ip_src_addr"]
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/files/source/pcap-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/pcap-config.json b/deployment/roles/metron_streaming/files/source/pcap-config.json
deleted file mode 100644
index 4b9c639..0000000
--- a/deployment/roles/metron_streaming/files/source/pcap-config.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "index": "pcap",
- "batchSize": 5,
- "enrichmentFieldMap":
- {
- "geo": ["ip_src_addr", "ip_dst_addr"],
- "host": ["ip_src_addr", "ip_dst_addr"]
- },
- "threatIntelFieldMap":
- {
- "ip": ["ip_src_addr", "ip_dst_addr"]
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/files/source/snort-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/snort-config.json b/deployment/roles/metron_streaming/files/source/snort-config.json
deleted file mode 100644
index 1208637..0000000
--- a/deployment/roles/metron_streaming/files/source/snort-config.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "index": "snort",
- "batchSize": 1,
- "enrichmentFieldMap":
- {
- "geo": ["ip_dst_addr", "ip_src_addr"],
- "host": ["host"]
- },
- "threatIntelFieldMap":
- {
- "ip": ["ip_dst_addr", "ip_src_addr"]
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/files/source/yaf-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/yaf-config.json b/deployment/roles/metron_streaming/files/source/yaf-config.json
deleted file mode 100644
index 65de961..0000000
--- a/deployment/roles/metron_streaming/files/source/yaf-config.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "index": "yaf",
- "batchSize": 5,
- "enrichmentFieldMap":
- {
- "geo": ["ip_dst_addr", "ip_src_addr"],
- "host": ["host"]
- },
- "threatIntelFieldMap":
- {
- "ip": ["ip_dst_addr", "ip_src_addr"]
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/handlers/main.yml b/deployment/roles/metron_streaming/handlers/main.yml
index 634d591..5aee8b4 100644
--- a/deployment/roles/metron_streaming/handlers/main.yml
+++ b/deployment/roles/metron_streaming/handlers/main.yml
@@ -15,5 +15,5 @@
# limitations under the License.
#
---
-- name: Load Source Config
- shell: java -cp {{ metron_directory }}/lib/{{ metron_jar_name }}::/usr/hdp/current/hadoop-client/lib/slf4j-api-1.7.10.jar org.apache.metron.utils.SourceConfigUtils -p {{ source_config_path }} -z {{ zookeeper_url }} && touch {{ source_config_path }}/configured
+- name: Load Config
+ shell: "{{ metron_directory }}/bin/zk_load_configs.sh -p {{ zookeeper_config_path }} -z {{ zookeeper_url }} && touch {{ zookeeper_config_path }}/configured"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
index 55ace91..863ecd8 100644
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -23,6 +23,16 @@
- { name: 'config'}
+- name: Copy Metron Solr bundle
+ copy:
+ src: "{{ metron_solr_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
+- name: Copy Metron Elasticsearch bundle
+ copy:
+ src: "{{ metron_elasticsearch_bundle_path }}"
+ dest: "{{ metron_directory }}"
+
- name: Copy Metron Topologies bundle
copy:
src: "{{ metron_topologies_bundle_path }}"
@@ -34,16 +44,17 @@
dest: "{{ metron_directory }}"
- name: Unbundle Metron bundles
- shell: cd {{ metron_directory }} && tar xzvf Metron-Topologies*.tar.gz && tar xzvf Metron-DataLoads*.tar.gz && rm *.tar.gz
-
-- name: Add hbase-site.xml to topology jar
- shell: cd {{ hbase_config_path }} && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} hbase-site.xml
+ shell: cd {{ metron_directory }} && tar xzvf Metron-Solr*.tar.gz && tar xzvf Metron-Elasticsearch*.tar.gz && tar xzvf Metron-Topologies*.tar.gz && tar xzvf Metron-DataLoads*.tar.gz && rm *.tar.gz
-- name: Add core-site.xml to topology jar
- shell: cd {{ hdfs_config_path }} && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} core-site.xml
-
-- name: Add hdfs-site.xml to topology jar
- shell: cd {{ hdfs_config_path }} && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} hdfs-site.xml
+- name: Add *-site.xml files to topology jars
+ shell: cd {{ item.config_path }} && jar -uf {{ metron_directory }}/lib/{{ item.jar_name }} {{ item.file_name }}
+ with_items:
+ - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hbase-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_solr_jar_name }}", file_name: "hdfs-site.xml" }
+ - { config_path: "{{ hbase_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hbase-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "core-site.xml" }
+ - { config_path: "{{ hdfs_config_path }}", jar_name: "{{ metron_elasticsearch_jar_name }}", file_name: "hdfs-site.xml" }
- name: Get Default mysql passowrd
include_vars: "../roles/mysql_server/defaults/main.yml"
@@ -55,9 +66,35 @@
- include: grok_upload.yml
run_once: true
-- name: Configure Metron topologies
+- name: Configure Metron Solr topologies
+ lineinfile: >
+ dest={{ metron_solr_properties_config_path }}
+ regexp="{{ item.regexp }}"
+ line="{{ item.line }}"
+ with_items:
+ - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+ - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
+ - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
+ - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
+ - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
+ - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
+ - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
+ - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
+ - { regexp: "bolt.hbase.table.name=", line: "bolt.hbase.table.name={{ pcap_hbase_table }}" }
+ - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
+ - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
+ - { regexp: "threat.intel.ip.table=", line: "threat.intel.ip.table={{ threatintel_ip_hbase_table }}" }
+ - { regexp: "threat.intel.ip.cf=", line: "threat.intel.ip.cf=t" }
+ - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
+ - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
+ - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
+ - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
+ - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
+ - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
+
+- name: Configure Metron Elasticsearch topologies
lineinfile: >
- dest={{ metron_properties_config_path }}
+ dest={{ metron_elasticsearch_properties_config_path }}
regexp="{{ item.regexp }}"
line="{{ item.line }}"
with_items:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/tasks/metron_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/metron_topology.yml b/deployment/roles/metron_streaming/tasks/metron_topology.yml
index 1da24f7..f8bf539 100644
--- a/deployment/roles/metron_streaming/tasks/metron_topology.yml
+++ b/deployment/roles/metron_streaming/tasks/metron_topology.yml
@@ -16,7 +16,14 @@
#
---
-- name: Submit Metron topologies
- command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_properties_config_path }} --remote {{ item }}
+- name: Submit Solr Metron topologies
+ command: storm jar {{ metron_directory }}/lib/{{ metron_solr_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_solr_properties_config_path }} --remote {{ item }}
with_items:
- "{{ storm_topologies }}"
+ - "{{ storm_topologies }}"
+ when: install_solr | default(False) == True
+
+- name: Submit Elasticsearch Metron topologies
+ command: storm jar {{ metron_directory }}/lib/{{ metron_elasticsearch_jar_name }} org.apache.storm.flux.Flux --filter {{ metron_elasticsearch_properties_config_path }} --remote {{ item }}
+ with_items:
+ - "{{ storm_topologies }}"
+ when: install_elasticsearch | default(False) == True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/tasks/source_config.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/source_config.yml b/deployment/roles/metron_streaming/tasks/source_config.yml
index 9233bac..959056c 100644
--- a/deployment/roles/metron_streaming/tasks/source_config.yml
+++ b/deployment/roles/metron_streaming/tasks/source_config.yml
@@ -17,15 +17,32 @@
---
- name: Create Source Config Directory
file:
- path: "{{ source_config_path }}"
+ path: "{{ zookeeper_config_path }}"
state: directory
-- name: Copy Source Config Files
+- name: Copy Elasticsearch Global Config File
+ template:
+ src: "templates/config/elasticsearch.global.json"
+ dest: "{{ zookeeper_global_config_path }}"
+ mode: 0644
+ when: install_elasticsearch | default(False) == True
+
+- name: Copy Solr Global Config File
+ template:
+ src: "../roles/metron_streaming/templates/config/solr.global.json"
+ dest: "{{ zookeeper_global_config_path }}"
+ mode: 0644
+ when: install_solr | default(False) == True
+
+- name: Copy Sensor Config Files
copy:
src: "{{ item }}"
- dest: "{{ source_config_path }}"
+ dest: "{{ zookeeper_config_path }}"
mode: 0644
- with_fileglob:
- - ../roles/metron_streaming/files/source/*.json
- notify: Load Source Config
+ with_items:
+ - ../roles/metron_streaming/files/config/
+ notify: Load Config
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/templates/config/elasticsearch.global.json b/deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
new file mode 100644
index 0000000..aa1076c
--- /dev/null
+++ b/deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
@@ -0,0 +1,6 @@
+{
+ "es.clustername": "{{ elasticsearch_cluster_name }}",
+ "es.ip": "{{ groups.search[0] }}",
+ "es.port": {{ elasticsearch_transport_port }},
+ "es.date.format": "yyyy.MM.dd.hh"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/templates/config/solr.global.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/templates/config/solr.global.json b/deployment/roles/metron_streaming/templates/config/solr.global.json
new file mode 100644
index 0000000..5cb7a4d
--- /dev/null
+++ b/deployment/roles/metron_streaming/templates/config/solr.global.json
@@ -0,0 +1,6 @@
+{
+ "solr.zookeeper": "{{ zookeeper_url }}",
+ "solr.collection": "{{ solr_collection_name }}",
+ "solr.numShards": {{ solr_number_shards }},
+ "solr.replicationFactor": {{ solr_replication_factor }}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/metron_streaming/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/vars/main.yml b/deployment/roles/metron_streaming/vars/main.yml
index b356762..f84b949 100644
--- a/deployment/roles/metron_streaming/vars/main.yml
+++ b/deployment/roles/metron_streaming/vars/main.yml
@@ -15,13 +15,23 @@
# limitations under the License.
#
---
-metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
-metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"
+metron_solr_jar_name: Metron-Solr-{{ metron_version }}.jar
+metron_elasticsearch_jar_name: Metron-Elasticsearch-{{ metron_version }}.jar
metron_directory: /usr/metron/{{ metron_version }}
metron_dataloads_name: Metron-DataLoads-{{ metron_version }}-archive.tar.gz
+metron_solr_bundle_name: Metron-Solr-{{ metron_version }}-archive.tar.gz
+metron_elasticsearch_bundle_name: Metron-Elasticsearch-{{ metron_version }}-archive.tar.gz
metron_topologies_bundle_name: Metron-Topologies-{{ metron_version }}-archive.tar.gz
metron_dataloads_path: "{{ playbook_dir }}/../../metron-streaming/Metron-DataLoads/target/{{ metron_dataloads_name }}"
+metron_solr_bundle_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Solr/target/{{ metron_solr_bundle_name }}"
+metron_elasticsearch_bundle_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Elasticsearch/target/{{ metron_elasticsearch_bundle_name }}"
metron_topologies_bundle_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_topologies_bundle_name }}"
metron_src_config_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs"
-metron_properties_config_path: "{{ metron_directory }}/config/etc/env/config.properties"
-
+metron_solr_properties_config_path: "{{ metron_directory }}/config/etc/env/solr.properties"
+metron_elasticsearch_properties_config_path: "{{ metron_directory }}/config/etc/env/elasticsearch.properties"
+elasticsearch_config_path: /etc/elasticsearch
+elasticsearch_cluster_name: metron
+elasticsearch_transport_port: 9300
+hbase_config_path: "/etc/hbase/conf"
+hdfs_config_path: "/etc/hadoop/conf"
+metron_hdfs_output_dir: "/apps/metron"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/solr/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/solr/defaults/main.yml b/deployment/roles/solr/defaults/main.yml
new file mode 100644
index 0000000..b40d534
--- /dev/null
+++ b/deployment/roles/solr/defaults/main.yml
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+rhel_hdp_utils_install_url: http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6/hdp-util.repo
+solr_install_path: /opt/lucidworks-hdpsearch/solr
+solr_user: solr
+solr_collection_name: Metron
+solr_config_dir: "{{ solr_install_path }}/server/solr/configsets/basic_configs/conf"
+solr_bin_dir: "/opt/lucidworks-hdpsearch/solr/bin"
+solr_config_name: "metron_conf"
+solr_number_shards: "{{ groups['search'] | length }}"
+solr_replication_factor: 1
+solr_autoSoftCommit_maxTime: 60
+solr_cmd: "{{ solr_bin_dir}}/solr create_collection -c {{ solr_collection_name }} -d {{ solr_config_dir }} -n {{ solr_config_name }} -shards {{ solr_number_shards }} -replicationFactor {{ solr_replication_factor }}"
+hdp_utils_repo_path: /etc/yum.repos.d/HDP-UTILS.repo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/solr/files/schema.xml
----------------------------------------------------------------------
diff --git a/deployment/roles/solr/files/schema.xml b/deployment/roles/solr/files/schema.xml
new file mode 100644
index 0000000..43452a2
--- /dev/null
+++ b/deployment/roles/solr/files/schema.xml
@@ -0,0 +1,191 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="metron" version="1.5">
+
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false"/>
+ <field name="sensorType" type="string" indexed="true" stored="true" required="true"/>;
+
+ <dynamicField name="*_i" type="int" indexed="true" stored="true"/>
+ <dynamicField name="*_is" type="int" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_s" type="string" indexed="true" stored="true"/>
+ <dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_l" type="long" indexed="true" stored="true"/>
+ <dynamicField name="*_ls" type="long" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_t" type="text_general" indexed="true" stored="true"/>
+ <dynamicField name="*_txt" type="text_general" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_en" type="text_en" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
+ <dynamicField name="*_bs" type="boolean" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_f" type="float" indexed="true" stored="true"/>
+ <dynamicField name="*_fs" type="float" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_d" type="double" indexed="true" stored="true"/>
+ <dynamicField name="*_ds" type="double" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false"/>
+ <dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
+ <dynamicField name="*_dts" type="date" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="*_p" type="location" indexed="true" stored="true"/>
+ <dynamicField name="*_ti" type="tint" indexed="true" stored="true"/>
+ <dynamicField name="*_tl" type="tlong" indexed="true" stored="true"/>
+ <dynamicField name="*_tf" type="tfloat" indexed="true" stored="true"/>
+ <dynamicField name="*_td" type="tdouble" indexed="true" stored="true"/>
+ <dynamicField name="*_tdt" type="tdate" indexed="true" stored="true"/>
+ <dynamicField name="*_c" type="currency" indexed="true" stored="true"/>
+ <dynamicField name="ignored_*" type="ignored" multiValued="true"/>
+ <dynamicField name="attr_*" type="text_general" indexed="true" stored="true" multiValued="true"/>
+ <dynamicField name="random_*" type="random"/>
+
+ <uniqueKey>id</uniqueKey>
+
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="binary" class="solr.BinaryField"/>
+ <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="lang/stopwords_en.txt"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.EnglishPossessiveFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="lang/stopwords_en.txt"
+ />
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.EnglishPossessiveFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="text_en_splitting" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
+ <analyzer type="index">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="lang/stopwords_en.txt"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory"
+ ignoreCase="true"
+ words="lang/stopwords_en.txt"
+ />
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.PorterStemFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="text_en_splitting_tight" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/>
+ <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+ <filter class="solr.EnglishMinimalStemFilterFactory"/>
+ <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="text_general_rev" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ReversedWildcardFilterFactory" withOriginal="true"
+ maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+ <analyzer>
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.TrimFilterFactory"/>
+ <filter class="solr.PatternReplaceFilterFactory"
+ pattern="([^a-z])" replacement="" replace="all"
+ />
+ </analyzer>
+ </fieldType>
+ <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField"/>
+ <fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/>
+ <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
+ <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
+ geo="true" distErrPct="0.025" maxDistErr="0.001" distanceUnits="kilometers"/>
+ <fieldType name="bbox" class="solr.BBoxField"
+ geo="true" distanceUnits="kilometers" numberType="_bbox_coord"/>
+ <fieldType name="_bbox_coord" class="solr.TrieDoubleField" precisionStep="8" docValues="true" stored="false"/>
+ <fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml"/>
+</schema>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/solr/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/solr/meta/main.yml b/deployment/roles/solr/meta/main.yml
new file mode 100644
index 0000000..454dd37
--- /dev/null
+++ b/deployment/roles/solr/meta/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
+ - java_jdk
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/solr/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/solr/tasks/main.yml b/deployment/roles/solr/tasks/main.yml
new file mode 100644
index 0000000..cfbb6b5
--- /dev/null
+++ b/deployment/roles/solr/tasks/main.yml
@@ -0,0 +1,74 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Check for Metron jar path
+ stat: path={{ hdp_utils_repo_path }}
+ register: hdp_utils
+
+
+- name: Install HDP-UTILs Repo
+ get_url:
+ url: "{{ rhel_hdp_utils_install_url }}"
+ dest: /etc/yum.repos.d/HDP-UTILS.repo
+ when: hdp_utils.stat.exists == False
+
+- name: Install HDP-UTIL gpg key
+ rpm_key:
+ state: present
+ key: http://pgp.mit.edu/pks/lookup?op=get&search=0xB9733A7A07513CAD
+ when: hdp_utils.stat.exists == False
+
+- name: Install Solr
+ yum:
+ name: lucidworks-hdpsearch
+ state: present
+
+- name: Create solr.xml from template
+ template:
+ src: solr.xml
+ dest: "{{ solr_install_path }}/server/solr"
+ mode: 0644
+ owner: "{{ solr_user }}"
+ group: "{{ solr_user }}"
+
+- name: Copy solrschema.xml to {{ inventory_hostname }}
+ copy:
+ src: schema.xml
+ dest: "{{ solr_config_dir }}"
+ mode: 0644
+ owner: "{{ solr_user }}"
+ group: "{{ solr_user }}"
+
+- name: Create solrconfig.xml from template
+ template:
+ src: solrconfig.xml
+ dest: "{{ solr_config_dir }}"
+ mode: 0644
+ owner: "{{ solr_user }}"
+ group: "{{ solr_user }}"
+
+- name: Start Solr
+ service:
+ name: solr
+ state: restarted
+ enabled: yes
+
+- name: Create Collection {{ solr_collection_name }} with {{ solr_number_shards }} shard(s) and replication factor {{ solr_replication_factor }}
+ shell: "{{ solr_cmd }}"
+ ignore_errors: yes
+ register: result
+ failed_when: result.rc == 1 and result.stderr.find("already exists!") == -1
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/solr/templates/solr.xml
----------------------------------------------------------------------
diff --git a/deployment/roles/solr/templates/solr.xml b/deployment/roles/solr/templates/solr.xml
new file mode 100644
index 0000000..407df13
--- /dev/null
+++ b/deployment/roles/solr/templates/solr.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ This is an example of a simple "solr.xml" file for configuring one or
+ more Solr Cores, as well as allowing Cores to be added, removed, and
+ reloaded via HTTP requests.
+
+ More information about options available in this configuration file,
+ and Solr Core administration can be found online:
+ http://wiki.apache.org/solr/CoreAdmin
+-->
+
+<solr>
+
+ <solrcloud>
+
+ <str name="host">${host:}</str>
+ <int name="hostPort">${jetty.port:8983}</int>
+ <str name="hostContext">${hostContext:solr}</str>
+
+ <bool name="genericCoreNodeNames">${genericCoreNodeNames:true}</bool>
+
+ <str name="zkHost">{{ zookeeper_url }}</str>
+ <int name="zkClientTimeout">${zkClientTimeout:30000}</int>
+ <int name="distribUpdateSoTimeout">${distribUpdateSoTimeout:600000}</int>
+ <int name="distribUpdateConnTimeout">${distribUpdateConnTimeout:60000}</int>
+
+ </solrcloud>
+
+ <shardHandlerFactory name="shardHandlerFactory"
+ class="HttpShardHandlerFactory">
+ <int name="socketTimeout">${socketTimeout:600000}</int>
+ <int name="connTimeout">${connTimeout:60000}</int>
+ </shardHandlerFactory>
+
+</solr>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/deployment/roles/solr/templates/solrconfig.xml
----------------------------------------------------------------------
diff --git a/deployment/roles/solr/templates/solrconfig.xml b/deployment/roles/solr/templates/solrconfig.xml
new file mode 100644
index 0000000..b00af0f
--- /dev/null
+++ b/deployment/roles/solr/templates/solrconfig.xml
@@ -0,0 +1,583 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ For more details about configurations options that may appear in
+ this file, see http://wiki.apache.org/solr/SolrConfigXml.
+-->
+<config>
+ <!-- In all configuration below, a prefix of "solr." for class names
+ is an alias that causes solr to search appropriate packages,
+ including org.apache.solr.(search|update|request|core|analysis)
+
+ You may also specify a fully qualified Java classname if you
+ have your own custom plugins.
+ -->
+
+ <!-- Controls what version of Lucene various components of Solr
+ adhere to. Generally, you want to use the latest version to
+ get all bug fixes and improvements. It is highly recommended
+ that you fully re-index after changing this setting as it can
+ affect both how text is indexed and queried.
+ -->
+ <luceneMatchVersion>5.2.1</luceneMatchVersion>
+
+ <!-- Data Directory
+
+ Used to specify an alternate directory to hold all index data
+ other than the default ./data under the Solr home. If
+ replication is in use, this should match the replication
+ configuration.
+ -->
+ <dataDir>${solr.data.dir:}</dataDir>
+
+
+ <!-- The DirectoryFactory to use for indexes.
+
+ solr.StandardDirectoryFactory is filesystem
+ based and tries to pick the best implementation for the current
+ JVM and platform. solr.NRTCachingDirectoryFactory, the default,
+ wraps solr.StandardDirectoryFactory and caches small files in memory
+ for better NRT performance.
+
+ One can force a particular implementation via solr.MMapDirectoryFactory,
+ solr.NIOFSDirectoryFactory, or solr.SimpleFSDirectoryFactory.
+
+ solr.RAMDirectoryFactory is memory based, not
+ persistent, and doesn't work with replication.
+ -->
+ <directoryFactory name="DirectoryFactory"
+ class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}">
+ </directoryFactory>
+
+ <!-- The CodecFactory for defining the format of the inverted index.
+ The default implementation is SchemaCodecFactory, which is the official Lucene
+ index format, but hooks into the schema to provide per-field customization of
+ the postings lists and per-document values in the fieldType element
+ (postingsFormat/docValuesFormat). Note that most of the alternative implementations
+ are experimental, so if you choose to customize the index format, it's a good
+ idea to convert back to the official format e.g. via IndexWriter.addIndexes(IndexReader)
+ before upgrading to a newer version to avoid unnecessary reindexing.
+ -->
+ <codecFactory class="solr.SchemaCodecFactory"/>
+
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ Index Config - These settings control low-level behavior of indexing
+ Most example settings here show the default value, but are commented
+ out, to more easily see where customizations have been made.
+
+ Note: This replaces <indexDefaults> and <mainIndex> from older versions
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
+ <indexConfig>
+
+ <!-- LockFactory
+
+ This option specifies which Lucene LockFactory implementation
+ to use.
+
+ single = SingleInstanceLockFactory - suggested for a
+ read-only index or when there is no possibility of
+ another process trying to modify the index.
+ native = NativeFSLockFactory - uses OS native file locking.
+ Do not use when multiple solr webapps in the same
+ JVM are attempting to share a single index.
+ simple = SimpleFSLockFactory - uses a plain file for locking
+
+ Defaults: 'native' is default for Solr3.6 and later, otherwise
+ 'simple' is the default
+
+ More details on the nuances of each LockFactory...
+ http://wiki.apache.org/lucene-java/AvailableLockFactories
+ -->
+ <lockType>${solr.lock.type:native}</lockType>
+
+ <!-- Lucene Infostream
+
+ To aid in advanced debugging, Lucene provides an "InfoStream"
+ of detailed information when indexing.
+
+ Setting the value to true will instruct the underlying Lucene
+ IndexWriter to write its info stream to solr's log. By default,
+ this is enabled here, and controlled through log4j.properties.
+ -->
+ <infoStream>true</infoStream>
+ </indexConfig>
+
+
+ <!-- JMX
+
+ This example enables JMX if and only if an existing MBeanServer
+ is found, use this if you want to configure JMX through JVM
+ parameters. Remove this to disable exposing Solr configuration
+ and statistics to JMX.
+
+ For more details see http://wiki.apache.org/solr/SolrJmx
+ -->
+ <jmx />
+ <!-- If you want to connect to a particular server, specify the
+ agentId
+ -->
+ <!-- <jmx agentId="myAgent" /> -->
+ <!-- If you want to start a new MBeanServer, specify the serviceUrl -->
+ <!-- <jmx serviceUrl="service:jmx:rmi:///jndi/rmi://localhost:9999/solr"/>
+ -->
+
+ <!-- The default high-performance update handler -->
+ <updateHandler class="solr.DirectUpdateHandler2">
+
+ <!-- Enables a transaction log, used for real-time get, durability, and
+ and solr cloud replica recovery. The log can grow as big as
+ uncommitted changes to the index, so use of a hard autoCommit
+ is recommended (see below).
+ "dir" - the target directory for transaction logs, defaults to the
+ solr data directory.
+ "numVersionBuckets" - sets the number of buckets used to keep
+ track of max version values when checking for re-ordered
+ updates; increase this value to reduce the cost of
+ synchronizing access to version buckets during high-volume
+ indexing, this requires 8 bytes (long) * numVersionBuckets
+ of heap space per Solr core.
+ -->
+ <updateLog>
+ <str name="dir">${solr.ulog.dir:}</str>
+ <int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int>
+ </updateLog>
+
+ <!-- AutoCommit
+
+ Perform a hard commit automatically under certain conditions.
+ Instead of enabling autoCommit, consider using "commitWithin"
+ when adding documents.
+
+ http://wiki.apache.org/solr/UpdateXmlMessages
+
+ maxDocs - Maximum number of documents to add since the last
+ commit before automatically triggering a new commit.
+
+ maxTime - Maximum amount of time in ms that is allowed to pass
+ since a document was added before automatically
+ triggering a new commit.
+ openSearcher - if false, the commit causes recent index changes
+ to be flushed to stable storage, but does not cause a new
+ searcher to be opened to make those changes visible.
+
+ If the updateLog is enabled, then it's highly recommended to
+ have some sort of hard autoCommit to limit the log size.
+ -->
+ <autoCommit>
+ <maxTime>${solr.autoCommit.maxTime:15000}</maxTime>
+ <openSearcher>false</openSearcher>
+ </autoCommit>
+
+ <!-- softAutoCommit is like autoCommit except it causes a
+ 'soft' commit which only ensures that changes are visible
+ but does not ensure that data is synced to disk. This is
+ faster and more near-realtime friendly than a hard commit.
+ -->
+ <autoSoftCommit>
+ <maxTime>${solr.autoSoftCommit.maxTime:{{ solr_autoSoftCommit_maxTime }}}</maxTime>
+ </autoSoftCommit>
+
+ </updateHandler>
+
+ <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ Query section - these settings control query time things like caches
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
+ <query>
+ <!-- Max Boolean Clauses
+
+ Maximum number of clauses in each BooleanQuery, an exception
+ is thrown if exceeded.
+
+ ** WARNING **
+
+ This option actually modifies a global Lucene property that
+ will affect all SolrCores. If multiple solrconfig.xml files
+ disagree on this property, the value at any given moment will
+ be based on the last SolrCore to be initialized.
+
+ -->
+ <maxBooleanClauses>1024</maxBooleanClauses>
+
+
+ <!-- Solr Internal Query Caches
+
+ There are two implementations of cache available for Solr,
+ LRUCache, based on a synchronized LinkedHashMap, and
+ FastLRUCache, based on a ConcurrentHashMap.
+
+ FastLRUCache has faster gets and slower puts in single
+ threaded operation and thus is generally faster than LRUCache
+ when the hit ratio of the cache is high (> 75%), and may be
+ faster under other scenarios on multi-cpu systems.
+ -->
+
+ <!-- Filter Cache
+
+ Cache used by SolrIndexSearcher for filters (DocSets),
+ unordered sets of *all* documents that match a query. When a
+ new searcher is opened, its caches may be prepopulated or
+ "autowarmed" using data from caches in the old searcher.
+ autowarmCount is the number of items to prepopulate. For
+ LRUCache, the autowarmed items will be the most recently
+ accessed items.
+
+ Parameters:
+ class - the SolrCache implementation LRUCache or
+ (LRUCache or FastLRUCache)
+ size - the maximum number of entries in the cache
+ initialSize - the initial capacity (number of entries) of
+ the cache. (see java.util.HashMap)
+ autowarmCount - the number of entries to prepopulate from
+ and old cache.
+ -->
+ <filterCache class="solr.FastLRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- Query Result Cache
+
+ Caches results of searches - ordered lists of document ids
+ (DocList) based on a query, a sort, and the range of documents requested.
+ Additional supported parameter by LRUCache:
+ maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
+ to occupy
+ -->
+ <queryResultCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- Document Cache
+
+ Caches Lucene Document objects (the stored fields for each
+ document). Since Lucene internal document ids are transient,
+ this cache will not be autowarmed.
+ -->
+ <documentCache class="solr.LRUCache"
+ size="512"
+ initialSize="512"
+ autowarmCount="0"/>
+
+ <!-- custom cache currently used by block join -->
+ <cache name="perSegFilter"
+ class="solr.search.LRUCache"
+ size="10"
+ initialSize="0"
+ autowarmCount="10"
+ regenerator="solr.NoOpRegenerator" />
+
+ <!-- Lazy Field Loading
+
+ If true, stored fields that are not requested will be loaded
+ lazily. This can result in a significant speed improvement
+ if the usual case is to not load all stored fields,
+ especially if the skipped fields are large compressed text
+ fields.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
+ <!-- Result Window Size
+
+ An optimization for use with the queryResultCache. When a search
+ is requested, a superset of the requested number of document ids
+ are collected. For example, if a search for a particular query
+ requests matching documents 10 through 19, and queryWindowSize is 50,
+ then documents 0 through 49 will be collected and cached. Any further
+ requests in that range can be satisfied via the cache.
+ -->
+ <queryResultWindowSize>20</queryResultWindowSize>
+
+ <!-- Maximum number of documents to cache for any entry in the
+ queryResultCache.
+ -->
+ <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+
+ <!-- Use Cold Searcher
+
+ If a search request comes in and there is no current
+ registered searcher, then immediately register the still
+ warming searcher and use it. If "false" then all requests
+ will block until the first searcher is done warming.
+ -->
+ <useColdSearcher>false</useColdSearcher>
+
+ <!-- Max Warming Searchers
+
+ Maximum number of searchers that may be warming in the
+ background concurrently. An error is returned if this limit
+ is exceeded.
+
+ Recommend values of 1-2 for read-only slaves, higher for
+ masters w/o cache warming.
+ -->
+ <maxWarmingSearchers>2</maxWarmingSearchers>
+
+ </query>
+
+
+ <!-- Request Dispatcher
+
+ This section contains instructions for how the SolrDispatchFilter
+ should behave when processing requests for this SolrCore.
+
+ handleSelect is a legacy option that affects the behavior of requests
+ such as /select?qt=XXX
+
+ handleSelect="true" will cause the SolrDispatchFilter to process
+ the request and dispatch the query to a handler specified by the
+ "qt" param, assuming "/select" isn't already registered.
+
+ handleSelect="false" will cause the SolrDispatchFilter to
+ ignore "/select" requests, resulting in a 404 unless a handler
+ is explicitly registered with the name "/select"
+
+ handleSelect="true" is not recommended for new users, but is the default
+ for backwards compatibility
+ -->
+ <requestDispatcher handleSelect="false" >
+ <!-- Request Parsing
+
+ These settings indicate how Solr Requests may be parsed, and
+ what restrictions may be placed on the ContentStreams from
+ those requests
+
+ enableRemoteStreaming - enables use of the stream.file
+ and stream.url parameters for specifying remote streams.
+
+ multipartUploadLimitInKB - specifies the max size (in KiB) of
+ Multipart File Uploads that Solr will allow in a Request.
+
+ formdataUploadLimitInKB - specifies the max size (in KiB) of
+ form data (application/x-www-form-urlencoded) sent via
+ POST. You can use POST to pass request parameters not
+ fitting into the URL.
+
+ addHttpRequestToContext - if set to true, it will instruct
+ the requestParsers to include the original HttpServletRequest
+ object in the context map of the SolrQueryRequest under the
+ key "httpRequest". It will not be used by any of the existing
+ Solr components, but may be useful when developing custom
+ plugins.
+
+ *** WARNING ***
+ The settings below authorize Solr to fetch remote files, You
+ should make sure your system has some authentication before
+ using enableRemoteStreaming="true"
+
+ -->
+ <requestParsers enableRemoteStreaming="true"
+ multipartUploadLimitInKB="2048000"
+ formdataUploadLimitInKB="2048"
+ addHttpRequestToContext="false"/>
+
+ <!-- HTTP Caching
+
+ Set HTTP caching related parameters (for proxy caches and clients).
+
+ The options below instruct Solr not to output any HTTP Caching
+ related headers
+ -->
+ <httpCaching never304="true" />
+
+ </requestDispatcher>
+
+ <!-- Request Handlers
+
+ http://wiki.apache.org/solr/SolrRequestHandler
+
+ Incoming queries will be dispatched to a specific handler by name
+ based on the path specified in the request.
+
+ Legacy behavior: If the request path uses "/select" but no Request
+ Handler has that name, and if handleSelect="true" has been specified in
+ the requestDispatcher, then the Request Handler is dispatched based on
+ the qt parameter. Handlers without a leading '/' are accessed this way
+ like so: http://host/app/[core/]select?qt=name If no qt is
+ given, then the requestHandler that declares default="true" will be
+ used or the one named "standard".
+
+ If a Request Handler is declared with startup="lazy", then it will
+ not be initialized until the first request that uses it.
+
+ -->
+ <!-- SearchHandler
+
+ http://wiki.apache.org/solr/SearchHandler
+
+ For processing Search Queries, the primary Request Handler
+ provided with Solr is "SearchHandler" It delegates to a sequent
+ of SearchComponents (see below) and supports distributed
+ queries across multiple shards
+ -->
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <!-- default values for query parameters can be specified, these
+ will be overridden by parameters in the request
+ -->
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <int name="rows">10</int>
+ </lst>
+
+ </requestHandler>
+
+ <!-- A request handler that returns indented JSON by default -->
+ <requestHandler name="/query" class="solr.SearchHandler">
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
+ <str name="df">text</str>
+ </lst>
+ </requestHandler>
+
+ <!--
+ The export request handler is used to export full sorted result sets.
+ Do not change these defaults.
+ -->
+ <requestHandler name="/export" class="solr.SearchHandler">
+ <lst name="invariants">
+ <str name="rq">{!xport}</str>
+ <str name="wt">xsort</str>
+ <str name="distrib">false</str>
+ </lst>
+
+ <arr name="components">
+ <str>query</str>
+ </arr>
+ </requestHandler>
+
+
+ <initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell">
+ <lst name="defaults">
+ <str name="df">text</str>
+ </lst>
+ </initParams>
+
+ <!-- Field Analysis Request Handler
+
+ RequestHandler that provides much the same functionality as
+ analysis.jsp. Provides the ability to specify multiple field
+ types and field names in the same request and outputs
+ index-time and query-time analysis for each of them.
+
+ Request parameters are:
+ analysis.fieldname - field name whose analyzers are to be used
+
+ analysis.fieldtype - field type whose analyzers are to be used
+ analysis.fieldvalue - text for index-time analysis
+ q (or analysis.q) - text for query time analysis
+ analysis.showmatch (true|false) - When set to true and when
+ query analysis is performed, the produced tokens of the
+ field value analysis will be marked as "matched" for every
+ token that is produces by the query analysis
+ -->
+ <requestHandler name="/analysis/field"
+ startup="lazy"
+ class="solr.FieldAnalysisRequestHandler" />
+
+
+ <!-- Document Analysis Handler
+
+ http://wiki.apache.org/solr/AnalysisRequestHandler
+
+ An analysis handler that provides a breakdown of the analysis
+ process of provided documents. This handler expects a (single)
+ content stream with the following format:
+
+ <docs>
+ <doc>
+ <field name="id">1</field>
+ <field name="name">The Name</field>
+ <field name="text">The Text Value</field>
+ </doc>
+ <doc>...</doc>
+ <doc>...</doc>
+ ...
+ </docs>
+
+ Note: Each document must contain a field which serves as the
+ unique key. This key is used in the returned response to associate
+ an analysis breakdown to the analyzed document.
+
+ Like the FieldAnalysisRequestHandler, this handler also supports
+ query analysis by sending either an "analysis.query" or "q"
+ request parameter that holds the query text to be analyzed. It
+ also supports the "analysis.showmatch" parameter which when set to
+ true, all field tokens that match the query tokens will be marked
+ as a "match".
+ -->
+ <requestHandler name="/analysis/document"
+ class="solr.DocumentAnalysisRequestHandler"
+ startup="lazy" />
+
+ <!-- Echo the request contents back to the client -->
+ <requestHandler name="/debug/dump" class="solr.DumpRequestHandler" >
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="echoHandler">true</str>
+ </lst>
+ </requestHandler>
+
+
+
+ <!-- Search Components
+
+ Search components are registered to SolrCore and used by
+ instances of SearchHandler (which can access them by name)
+
+ By default, the following components are available:
+
+ <searchComponent name="query" class="solr.QueryComponent" />
+ <searchComponent name="facet" class="solr.FacetComponent" />
+ <searchComponent name="mlt" class="solr.MoreLikeThisComponent" />
+ <searchComponent name="highlight" class="solr.HighlightComponent" />
+ <searchComponent name="stats" class="solr.StatsComponent" />
+ <searchComponent name="debug" class="solr.DebugComponent" />
+
+ -->
+
+ <!-- Terms Component
+
+ http://wiki.apache.org/solr/TermsComponent
+
+ A component to return terms and document frequency of those
+ terms
+ -->
+ <searchComponent name="terms" class="solr.TermsComponent"/>
+
+ <!-- A request handler for demonstrating the terms component -->
+ <requestHandler name="/terms" class="solr.SearchHandler" startup="lazy">
+ <lst name="defaults">
+ <bool name="terms">true</bool>
+ <bool name="distrib">false</bool>
+ </lst>
+ <arr name="components">
+ <str>terms</str>
+ </arr>
+ </requestHandler>
+
+ <!-- Legacy config for the admin interface -->
+ <admin>
+ <defaultQuery>*:*</defaultQuery>
+ </admin>
+
+</config>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/metron-streaming/Metron-Common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/pom.xml b/metron-streaming/Metron-Common/pom.xml
index 605c7ed..22460d0 100644
--- a/metron-streaming/Metron-Common/pom.xml
+++ b/metron-streaming/Metron-Common/pom.xml
@@ -175,6 +175,12 @@
<groupId>org.apache.storm</groupId>
<artifactId>storm-kafka</artifactId>
<version>${global_storm_version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>org.apache.curator</artifactId>
+ <groupId>curator-client</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java
index c6eafe9..78efa5c 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/Constants.java
@@ -19,9 +19,16 @@ package org.apache.metron;
public class Constants {
+ public static final String GLOBAL_CONFIG_NAME = "global";
+ public static final String SENSORS_CONFIG_NAME = "sensors";
public static final String ZOOKEEPER_ROOT = "/metron";
public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology";
- public static final String SOURCE_TYPE = "source.type";
+ public static final String ZOOKEEPER_GLOBAL_ROOT = ZOOKEEPER_TOPOLOGY_ROOT + "/" + GLOBAL_CONFIG_NAME;
+ public static final String ZOOKEEPER_SENSOR_ROOT = ZOOKEEPER_TOPOLOGY_ROOT + "/" + SENSORS_CONFIG_NAME;
+ public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000;
+ public static final String SENSOR_TYPE = "source.type";
public static final String ENRICHMENT_TOPIC = "enrichments";
public static final String ERROR_STREAM = "error";
+
}
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/e59b1a31/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
index e9a9237..1f12f7a 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
@@ -22,9 +22,8 @@ import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
-import backtype.storm.tuple.Values;
import org.apache.metron.Constants;
-import org.apache.metron.domain.SourceConfig;
+import org.apache.metron.domain.SensorEnrichmentConfig;
import org.apache.metron.helpers.topology.ErrorUtils;
import org.apache.metron.topology.TopologyUtils;
import org.apache.metron.writer.interfaces.BulkMessageWriter;
@@ -36,14 +35,12 @@ import java.util.*;
public class BulkMessageWriterBolt extends ConfiguredBolt {
- int count = 0;
-
private static final Logger LOG = LoggerFactory
.getLogger(BulkMessageWriterBolt.class);
private OutputCollector collector;
private BulkMessageWriter<JSONObject> bulkMessageWriter;
- private Map<String, List<Tuple>> sourceTupleMap = new HashMap<>();
- private Map<String, List<JSONObject>> sourceMessageMap = new HashMap<>();
+ private Map<String, List<Tuple>> sensorTupleMap = new HashMap<>();
+ private Map<String, List<JSONObject>> sensorMessageMap = new HashMap<>();
public BulkMessageWriterBolt(String zookeeperUrl) {
super(zookeeperUrl);
@@ -58,7 +55,11 @@ public class BulkMessageWriterBolt extends ConfiguredBolt {
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
super.prepare(stormConf, context, collector);
- bulkMessageWriter.init(stormConf);
+ try {
+ bulkMessageWriter.init(stormConf, configurations);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
@SuppressWarnings("unchecked")
@@ -66,23 +67,22 @@ public class BulkMessageWriterBolt extends ConfiguredBolt {
public void execute(Tuple tuple) {
JSONObject message = (JSONObject)((JSONObject) tuple.getValueByField("message")).clone();
message.put("index." + bulkMessageWriter.getClass().getSimpleName().toLowerCase() + ".ts", "" + System.currentTimeMillis());
- String sourceType = TopologyUtils.getSourceType(message);
- SourceConfig configuration = configurations.get(sourceType);
- int batchSize = configuration != null ? configuration.getBatchSize() : 1;
- List<Tuple> tupleList = sourceTupleMap.get(sourceType);
+ String sensorType = TopologyUtils.getSensorType(message);
+ SensorEnrichmentConfig sensorEnrichmentConfig = configurations.getSensorEnrichmentConfig(sensorType);
+ int batchSize = sensorEnrichmentConfig != null ? sensorEnrichmentConfig.getBatchSize() : 1;
+ List<Tuple> tupleList = sensorTupleMap.get(sensorType);
if (tupleList == null) tupleList = new ArrayList<>();
tupleList.add(tuple);
- List<JSONObject> messageList = sourceMessageMap.get(sourceType);
+ List<JSONObject> messageList = sensorMessageMap.get(sensorType);
if (messageList == null) messageList = new ArrayList<>();
messageList.add(message);
if (messageList.size() < batchSize) {
- sourceTupleMap.put(sourceType, tupleList);
- sourceMessageMap.put(sourceType, messageList);
+ sensorTupleMap.put(sensorType, tupleList);
+ sensorMessageMap.put(sensorType, messageList);
} else {
try {
-
- String esType = sourceType + "_doc";
- bulkMessageWriter.write(esType, configuration, tupleList, messageList);
+ String esType = sensorType + "_doc";
+ bulkMessageWriter.write(esType, configurations, tupleList, messageList);
for(Tuple t: tupleList) {
collector.ack(t);
}
@@ -92,8 +92,8 @@ public class BulkMessageWriterBolt extends ConfiguredBolt {
}
ErrorUtils.handleError(collector, e, Constants.ERROR_STREAM);
}
- sourceTupleMap.remove(sourceType);
- sourceMessageMap.remove(sourceType);
+ sensorTupleMap.remove(sensorType);
+ sensorMessageMap.remove(sensorType);
}
}