You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by le...@apache.org on 2018/06/26 21:15:28 UTC
[12/53] [abbrv] metron git commit: METRON-1423 Ambari work to handle
Solr configuration (merrimanr) closes apache/metron#934
METRON-1423 Ambari work to handle Solr configuration (merrimanr) closes apache/metron#934
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/84cac7e8
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/84cac7e8
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/84cac7e8
Branch: refs/heads/master
Commit: 84cac7e85f6b8421063f329d647be9bc82ea5f4c
Parents: 1006d9d
Author: merrimanr <me...@gmail.com>
Authored: Wed Feb 28 16:37:30 2018 -0600
Committer: merrimanr <me...@gmail.com>
Committed: Wed Feb 28 16:37:30 2018 -0600
----------------------------------------------------------------------
.../ansible/playbooks/metron_full_install.yml | 4 ++
.../ansible/playbooks/solr_install.yml | 40 ++++++++++++
.../roles/ambari_config/vars/single_node_vm.yml | 1 +
.../ansible/roles/solr/defaults/main.yml | 23 +++++++
.../ansible/roles/solr/meta/main.yml | 19 ++++++
.../ansible/roles/solr/tasks/main.yml | 36 +++++++++++
metron-deployment/packaging/ambari/.gitignore | 1 +
.../packaging/ambari/metron-mpack/pom.xml | 8 ++-
.../METRON/CURRENT/configuration/metron-env.xml | 6 ++
.../configuration/metron-indexing-env.xml | 32 +++++++---
.../common-services/METRON/CURRENT/metainfo.xml | 4 ++
.../package/scripts/indexing_commands.py | 25 +++++++-
.../CURRENT/package/scripts/indexing_master.py | 67 ++++++++++++++++----
.../CURRENT/package/scripts/metron_service.py | 5 ++
.../package/scripts/params/params_linux.py | 15 ++++-
.../package/scripts/params/status_params.py | 3 +
.../METRON/CURRENT/themes/metron_theme.json | 18 ++++--
.../docker/rpm-docker/SPECS/metron.spec | 4 ++
.../src/main/config/elasticsearch.properties.j2 | 2 +-
metron-platform/metron-solr/README.md | 19 ++----
.../src/main/config/solr.properties.j2 | 2 +-
.../src/main/scripts/create_collection.sh | 27 ++++++++
.../src/main/scripts/delete_collection.sh | 27 ++++++++
.../src/main/scripts/install_solr.sh | 2 +-
.../metron-solr/src/main/scripts/start_solr.sh | 27 ++++++++
.../metron-solr/src/main/scripts/stop_solr.sh | 27 ++++++++
pom.xml | 2 +-
27 files changed, 401 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/playbooks/metron_full_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/metron_full_install.yml b/metron-deployment/ansible/playbooks/metron_full_install.yml
index b517671..92a6473 100644
--- a/metron-deployment/ansible/playbooks/metron_full_install.yml
+++ b/metron-deployment/ansible/playbooks/metron_full_install.yml
@@ -25,6 +25,10 @@
tags:
- build
+- include: solr_install.yml
+ tags:
+ - solr
+
- include: ambari_install.yml
tags:
- ambari
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/playbooks/solr_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/solr_install.yml b/metron-deployment/ansible/playbooks/solr_install.yml
new file mode 100644
index 0000000..464ec80
--- /dev/null
+++ b/metron-deployment/ansible/playbooks/solr_install.yml
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- hosts: ec2
+ become: true
+ tasks:
+ - include_vars: ../../amazon-ec2/conf/defaults.yml
+ tags:
+ - ec2
+
+- hosts: packer
+ become: true
+ tasks:
+ - include_vars: ../../development/centos6/ansible/inventory/group_vars/all
+ tags:
+ - packer
+
+#
+# Solr
+#
+- hosts: search
+ become: true
+ roles:
+ - { role: solr, tags: ['solr'] }
+ tags:
+ - solr
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index bf54fe0..9addc5e 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -116,6 +116,7 @@ required_configurations:
storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744"
es_hosts: "{{ groups.search | join(',') }}"
zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
+ solr_zookeeper_url: "{{ groups.search[0] }}:9983"
- metron-rest-env:
metron_jdbc_driver: "org.h2.Driver"
metron_jdbc_url: "jdbc:h2:file:~/metrondb"
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/roles/solr/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/solr/defaults/main.yml b/metron-deployment/ansible/roles/solr/defaults/main.yml
new file mode 100644
index 0000000..61562f8
--- /dev/null
+++ b/metron-deployment/ansible/roles/solr/defaults/main.yml
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+solr_version: 6.6.2
+solr_installation_user: "solr"
+solr_user_home: /var/solr
+solr_home: "{{ solr_user_home }}/solr-{{ solr_version }}"
+solr_url: http://archive.apache.org/dist/lucene/solr/{{ solr_version }}/solr-{{ solr_version }}.tgz
+
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/roles/solr/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/solr/meta/main.yml b/metron-deployment/ansible/roles/solr/meta/main.yml
new file mode 100644
index 0000000..ddf6aa9
--- /dev/null
+++ b/metron-deployment/ansible/roles/solr/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/ansible/roles/solr/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/solr/tasks/main.yml b/metron-deployment/ansible/roles/solr/tasks/main.yml
new file mode 100644
index 0000000..cf47177
--- /dev/null
+++ b/metron-deployment/ansible/roles/solr/tasks/main.yml
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Solr user
+ user:
+ name: solr
+ home: "{{ solr_user_home }}"
+ system: yes
+
+- name: Download Solr
+ get_url:
+ url: "{{ solr_url }}"
+ dest: "{{ solr_user_home }}/solr-{{ solr_version }}.tar.gz"
+
+- name: Extract Solr tarball
+ unarchive:
+ src: "{{ solr_user_home }}/solr-{{ solr_version }}.tar.gz"
+ dest: "{{ solr_user_home }}"
+ copy: no
+ creates: "{{ solr_home }}"
+ owner: "{{ solr_installation_user }}"
+ group: "{{ solr_installation_user }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore
index ca2e75c..72538a0 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -1,5 +1,6 @@
archive.zip
*.hash
elasticsearch.properties.j2
+solr.properties.j2
hdfs.properties.j2
enrichment.properties.j2
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 491e8dd..e32063b 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -128,7 +128,13 @@
</includes>
<filtering>false</filtering>
</resource>
-
+ <resource>
+ <directory>${basedir}/../../../../metron-platform/metron-solr/src/main/config</directory>
+ <includes>
+ <include>solr.properties.j2</include>
+ </includes>
+ <filtering>false</filtering>
+ </resource>
</resources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 87866e8..8a52583 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -100,6 +100,12 @@
<description>Elasticsearch HTTP port. (9200)</description>
<display-name>Elasticsearch HTTP port</display-name>
</property>
+ <property>
+ <name>solr_zookeeper_url</name>
+ <value>{{zookeeper_quorum}}</value>
+ <description>Comma delimited list of Zookeeper Urls: zkhost1:2181,zkhost1:2181</description>
+ <display-name>Solr Zookeeper Urls</display-name>
+ </property>
<property require-input = "true">
<name>storm_rest_addr</name>
<display-name>Storm Rest Server Address</display-name>
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
index b960536..3d268b7 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-indexing-env.xml
@@ -22,7 +22,7 @@
<name>ra_indexing_kafka_start</name>
<description>Indexing Topology Spout Offset</description>
<value>UNCOMMITTED_EARLIEST</value>
- <display-name>Elasticsearch Indexing Offset</display-name>
+ <display-name>Random Access Indexing Offset</display-name>
<value-attributes>
<type>value-list</type>
<entries>
@@ -80,10 +80,22 @@
<display-name>Indexing Error Topic</display-name>
</property>
<property>
- <name>ra_indexing_writer_class_name</name>
- <description>Indexing Writer Class Name</description>
- <value>org.apache.metron.elasticsearch.writer.ElasticsearchWriter</value>
- <display-name>Indexing Writer Class Name</display-name>
+ <name>ra_indexing_writer</name>
+ <description>Search Engine Used for Random Access</description>
+ <value>Elasticsearch</value>
+ <display-name>Random Access Search Engine</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>Elasticsearch</value>
+ </entry>
+ <entry>
+ <value>Solr</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
</property>
<property>
<name>update_hbase_table</name>
@@ -101,7 +113,7 @@
<name>ra_indexing_workers</name>
<description>Number of Indexing Topology Workers</description>
<value>1</value>
- <display-name>Indexing Workers for Elasticsearch</display-name>
+ <display-name>Indexing Workers for Random Access</display-name>
</property>
<property>
<name>batch_indexing_workers</name>
@@ -113,7 +125,7 @@
<name>ra_indexing_acker_executors</name>
<description>Number of Indexing Topology Ackers</description>
<value>1</value>
- <display-name>Enrichment Ackers for Elasticsearch</display-name>
+ <display-name>Enrichment Ackers for Random Access</display-name>
</property>
<property>
<name>batch_indexing_acker_executors</name>
@@ -135,7 +147,7 @@
<name>ra_indexing_topology_max_spout_pending</name>
<description>Indexing Topology Spout Max Pending Tuples</description>
<value/>
- <display-name>Indexing Max Pending for Elasticsearch</display-name>
+ <display-name>Indexing Max Pending for Random Access</display-name>
<value-attributes>
<empty-value-valid>true</empty-value-valid>
</value-attributes>
@@ -152,7 +164,7 @@
<property>
<name>ra_indexing_kafka_spout_parallelism</name>
- <description>Indexing Topology Kafka Spout Parallelism for Elasticsearch</description>
+ <description>Indexing Topology Kafka Spout Parallelism for Random Access</description>
<value>1</value>
<display-name>Indexing Spout Parallelism</display-name>
</property>
@@ -164,7 +176,7 @@
</property>
<property>
<name>ra_indexing_writer_parallelism</name>
- <description>Indexing Topology Writer Bolt Parallelism for Elasticsearch</description>
+ <description>Indexing Topology Writer Bolt Parallelism for Random Access</description>
<value>1</value>
<display-name>Indexing Writer Parallelism</display-name>
</property>
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 2b5b405..ff2d7d6 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -267,6 +267,7 @@
<scriptType>PYTHON</scriptType>
</commandScript>
<configuration-dependencies>
+ <config-type>metron-indexing-env</config-type>
<config-type>metron-rest-env</config-type>
</configuration-dependencies>
</component>
@@ -396,6 +397,9 @@
<name>metron-elasticsearch</name>
</package>
<package>
+ <name>metron-solr</name>
+ </package>
+ <package>
<name>metron-pcap</name>
</package>
<package>
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 4c862f0..7af0f51 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -54,6 +54,7 @@ class IndexingCommands:
self.__hbase_configured = os.path.isfile(self.__params.indexing_hbase_configured_flag_file)
self.__hbase_acl_configured = os.path.isfile(self.__params.indexing_hbase_acl_configured_flag_file)
self.__elasticsearch_template_installed = os.path.isfile(self.__params.elasticsearch_template_installed_flag_file)
+ self.__solr_schema_installed = os.path.isfile(self.__params.solr_schema_installed_flag_file)
self.__hdfs_perm_configured = os.path.isfile(self.__params.indexing_hdfs_perm_configured_flag_file)
def __get_topics(self):
@@ -78,6 +79,21 @@ class IndexingCommands:
"metaalert_index": params.meta_index_path
}
+ def get_solr_schemas(self):
+ """
+ Defines the Solr schemas.
+ :return: Dict where key is the name of a collection and the
+ value is a path to file containing the schema definition.
+ """
+ from params import params
+ return {
+ "bro": params.bro_schema_path,
+ "yaf": params.yaf_schema_path,
+ "snort": params.snort_schema_path,
+ "error": params.error_schema_path,
+ "metaalert": params.meta_schema_path
+ }
+
def is_configured(self):
return self.__configured
@@ -96,6 +112,9 @@ class IndexingCommands:
def is_elasticsearch_template_installed(self):
return self.__elasticsearch_template_installed
+ def is_solr_schema_installed(self):
+ return self.__solr_schema_installed
+
def set_configured(self):
metron_service.set_configured(self.__params.metron_user, self.__params.indexing_configured_flag_file, "Setting Indexing configured to True")
@@ -114,6 +133,9 @@ class IndexingCommands:
def set_elasticsearch_template_installed(self):
metron_service.set_configured(self.__params.metron_user, self.__params.elasticsearch_template_installed_flag_file, "Setting Elasticsearch template installed to True")
+ def set_solr_schema_installed(self):
+ metron_service.set_configured(self.__params.metron_user, self.__params.solr_schema_installed_flag_file, "Setting Solr schema installed to True")
+
def create_hbase_tables(self):
Logger.info("Creating HBase Tables for indexing")
metron_service.create_hbase_table(self.__params,
@@ -206,8 +228,9 @@ class IndexingCommands:
self.__params.metron_keytab_path,
self.__params.metron_principal_name,
execute_user=self.__params.metron_user)
-
start_cmd_template = """{0}/bin/start_elasticsearch_topology.sh"""
+ if self.__params.ra_indexing_writer == 'Solr':
+ start_cmd_template = """{0}/bin/start_solr_topology.sh"""
start_cmd = start_cmd_template.format(self.__params.metron_home)
Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index ce8c074..e63ea2d 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -49,6 +49,11 @@ class Indexing(Script):
owner=params.metron_user,
group=params.metron_group
)
+ File(format("{metron_config_path}/solr.properties"),
+ content=Template("solr.properties.j2"),
+ owner=params.metron_user,
+ group=params.metron_group
+ )
File(format("{metron_config_path}/hdfs.properties"),
content=Template("hdfs.properties.j2"),
owner=params.metron_user,
@@ -87,17 +92,29 @@ class Indexing(Script):
env.set_params(params)
self.configure(env)
commands = IndexingCommands(params)
-
- # Install elasticsearch templates
- try:
- if not commands.is_elasticsearch_template_installed():
- self.elasticsearch_template_install(env)
- commands.set_elasticsearch_template_installed()
-
- except Exception as e:
- msg = "WARNING: Elasticsearch index templates could not be installed. " \
- "Is Elasticsearch running? Will reattempt install on next start. error={0}"
- Logger.warning(msg.format(e))
+ if params.ra_indexing_writer == 'Solr':
+ Logger.info("Loading Solr schemas")
+ # Install Solr schemas
+ try:
+ if not commands.is_solr_schema_installed():
+ self.solr_schema_install(env)
+ commands.set_solr_schema_installed()
+
+ except Exception as e:
+ msg = "WARNING: Solr schemas could not be installed. " \
+ "Is Solr running? Will reattempt install on next start. error={0}"
+ Logger.warning(msg.format(e))
+ else:
+ # Install elasticsearch templates
+ try:
+ if not commands.is_elasticsearch_template_installed():
+ self.elasticsearch_template_install(env)
+ commands.set_elasticsearch_template_installed()
+
+ except Exception as e:
+ msg = "WARNING: Elasticsearch index templates could not be installed. " \
+ "Is Elasticsearch running? Will reattempt install on next start. error={0}"
+ Logger.warning(msg.format(e))
commands.start_indexing_topology(env)
@@ -149,6 +166,34 @@ class Indexing(Script):
cmd.format(params.es_http_url, template_name),
logoutput=True)
+ def solr_schema_install(self, env):
+ from params import params
+ env.set_params(params)
+ Logger.info("Installing Solr schemas")
+
+ commands = IndexingCommands(params)
+ for collection_name, config_path in commands.get_solr_schemas().iteritems():
+
+ # install the schema
+
+ cmd = "{0}/bin/solr create -c {1} -d {2}"
+ Execute(
+ cmd.format(params.solr_home, collection_name, config_path),
+ logoutput=True, user="solr")
+
+ def solr_schema_delete(self, env):
+ from params import params
+ env.set_params(params)
+ Logger.info("Deleting Solr schemas")
+
+ commands = IndexingCommands(params)
+ for collection_name, config_path in commands.get_solr_schemas().iteritems():
+ # delete the schema
+ cmd = "{0}/bin/solr delete -c {1}"
+ Execute(
+ cmd.format(params.solr_home, collection_name),
+ logoutput=True, user="solr")
+
def zeppelin_notebook_import(self, env):
from params import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index 330d3c0..1f7ec6b 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -72,6 +72,11 @@ def build_global_config_patch(params, patch_file):
},
{
"op": "add",
+ "path": "/solr.zookeeper",
+ "value": "{{solr_zookeeper_url}}"
+ },
+ {
+ "op": "add",
"path": "/parser.error.topic",
"value": "{{parser_error_topic}}"
},
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 3506fab..5bfa1dc 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -84,6 +84,7 @@ indexing_hbase_configured_flag_file = status_params.indexing_hbase_configured_fl
indexing_hbase_acl_configured_flag_file = status_params.indexing_hbase_acl_configured_flag_file
indexing_hdfs_perm_configured_flag_file = status_params.indexing_hdfs_perm_configured_flag_file
elasticsearch_template_installed_flag_file = status_params.elasticsearch_template_installed_flag_file
+solr_schema_installed_flag_file = status_params.solr_schema_installed_flag_file
global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
# Elasticsearch hosts and port management
@@ -120,6 +121,11 @@ if has_zk_host:
# last port config
zookeeper_quorum += ':' + zookeeper_clientPort
+# Solr params
+solr_version = '6.6.2'
+solr_home = '/var/solr/solr-' + solr_version
+solr_zookeeper_url = format(format(config['configurations']['metron-env']['solr_zookeeper_url']))
+
# Storm
storm_rest_addr = status_params.storm_rest_addr
@@ -200,6 +206,13 @@ yaf_index_path = tmp_dir + "/yaf_index.template"
error_index_path = tmp_dir + "/error_index.template"
meta_index_path = tmp_dir + "/metaalert_index.template"
+# Solr Schemas
+bro_schema_path = metron_home + "/config/schema/bro"
+snort_schema_path = metron_home + "/config/schema/snort"
+yaf_schema_path = metron_home + "/config/schema/yaf"
+error_schema_path = metron_home + "/config/schema/error"
+meta_schema_path = metron_home + "/config/schema/metaalert"
+
# Zeppelin Notebooks
metron_config_zeppelin_path = format("{metron_config_path}/zeppelin")
zeppelin_shiro_ini_content = status_params.zeppelin_shiro_ini_content
@@ -305,7 +318,7 @@ indexing_input_topic = status_params.indexing_input_topic
indexing_error_topic = config['configurations']['metron-indexing-env']['indexing_error_topic']
metron_random_access_indexing_topology = status_params.metron_random_access_indexing_topology
metron_batch_indexing_topology = status_params.metron_batch_indexing_topology
-ra_indexing_writer_class_name = config['configurations']['metron-indexing-env']['ra_indexing_writer_class_name']
+ra_indexing_writer = config['configurations']['metron-indexing-env']['ra_indexing_writer']
batch_indexing_writer_class_name = config['configurations']['metron-indexing-env']['batch_indexing_writer_class_name']
ra_indexing_workers = config['configurations']['metron-indexing-env']['ra_indexing_workers']
batch_indexing_workers = config['configurations']['metron-indexing-env']['batch_indexing_workers']
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 62cfc7a..1e5c35f 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -79,6 +79,9 @@ indexing_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../me
# Elasticsearch
elasticsearch_template_installed_flag_file = metron_zookeeper_config_path + '/../metron_elasticsearch_template_installed_flag_file'
+# Solr
+solr_schema_installed_flag_file = metron_zookeeper_config_path + '/../metron_solr_schema_installed_flag_file'
+
# REST
metron_rest_port = config['configurations']['metron-rest-env']['metron_rest_port']
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index cef9a3b..ae15c31 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -183,7 +183,7 @@
"subsections": [
{
"name": "subsection-indexing-storm",
- "display-name": "Index Writer - Elasticsearch",
+ "display-name": "Index Writer - Random Access",
"row-index": "0",
"column-index": "0",
"row-span": "1",
@@ -391,6 +391,10 @@
"subsection-name": "subsection-index-settings"
},
{
+ "config": "metron-env/solr_zookeeper_url",
+ "subsection-name": "subsection-index-settings"
+ },
+ {
"config": "metron-parsers-env/parsers",
"subsection-name": "subsection-parsers"
},
@@ -503,7 +507,7 @@
"subsection-name": "subsection-indexing-update"
},
{
- "config": "metron-indexing-env/ra_indexing_writer_class_name",
+ "config": "metron-indexing-env/ra_indexing_writer",
"subsection-name": "subsection-indexing-storm"
},
{
@@ -711,6 +715,12 @@
}
},
{
+ "config": "metron-env/solr_zookeeper_url",
+ "widget": {
+ "type": "text-field"
+ }
+ },
+ {
"config": "metron-parsers-env/parsers",
"widget": {
"type": "text-field"
@@ -888,9 +898,9 @@
}
},
{
- "config": "metron-indexing-env/ra_indexing_writer_class_name",
+ "config": "metron-indexing-env/ra_indexing_writer",
"widget": {
- "type": "text-field"
+ "type": "combo"
}
},
{
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 5f97f62..5222efc 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -225,8 +225,12 @@ This package installs the Metron Solr files
%dir %{metron_home}/bin
%dir %{metron_home}/config
%dir %{metron_home}/lib
+%{metron_home}/bin/create_collection.sh
+%{metron_home}/bin/delete_collection.sh
%{metron_home}/bin/install_solr.sh
+%{metron_home}/bin/start_solr.sh
%{metron_home}/bin/start_solr_topology.sh
+%{metron_home}/bin/stop_solr.sh
%{metron_home}/config/solr.properties
%{metron_home}/config/schema/bro/managed-schema
%{metron_home}/config/schema/bro/solrconfig.xml
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2 b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2
index 00ad9dc..023ba2f 100644
--- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2
+++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties.j2
@@ -35,7 +35,7 @@ indexing.input.topic={{indexing_input_topic}}
indexing.error.topic={{indexing_error_topic}}
##### Indexing #####
-indexing.writer.class.name={{ra_indexing_writer_class_name}}
+indexing.writer.class.name=org.apache.metron.elasticsearch.writer.ElasticsearchWriter
##### Parallelism #####
kafka.spout.parallelism={{ra_indexing_kafka_spout_parallelism}}
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md
index 164160e..4321114 100644
--- a/metron-platform/metron-solr/README.md
+++ b/metron-platform/metron-solr/README.md
@@ -72,10 +72,10 @@ via the global config. The following settings are possible as part of the globa
## Installing
-A script is provided in the installation for installing Solr Cloud in quick-start mode in the [full dev environment for CentOS](../../metron-deployment/development/centos6).
-The script is installed as part of the Solr RPM. Metron's Ambari MPack does not currently manage Solr installation in any way, so
-you must first install the RPM in order to run the Solr setup script.
+Solr is installed in the [full dev environment for CentOS](../../metron-deployment/development/centos6) by default but is not started initially. Navigate to `$METRON_HOME/bin`
+and start Solr Cloud by running `start_solr.sh`.
+Metron's Ambari MPack installs several scripts in `$METRON_HOME/bin` that can be used to manage Solr. A script is also provided for installing Solr Cloud outside of full dev.
The script performs the following tasks
* Stops ES and Kibana
@@ -85,16 +85,9 @@ The script performs the following tasks
_Note: for details on setting up Solr Cloud in production mode, see https://lucene.apache.org/solr/guide/6_6/taking-solr-to-production.html_
-Login to the full dev environment as root and execute the following to install the Solr RPM.
-
-```
-rpm -ivh /localrepo/metron-solr-*.rpm
-```
-
-This will lay down the necessary files to setup Solr Cloud. Navigate to `$METRON_HOME/bin` and spin up Solr Cloud by running `install_solr.sh`.
-
-After running this script, Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at http://localhost:8983/solr/#/~cloud. This manner
-of starting Solr will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html)
+Navigate to `$METRON_HOME/bin` and spin up Solr Cloud by running `install_solr.sh`. After running this script,
+Elasticsearch and Kibana will have been stopped and you should now have an instance of Solr Cloud up and running at http://localhost:8983/solr/#/~cloud. This manner of starting Solr
+will also spin up an embedded Zookeeper instance at port 9983. More information can be found [here](https://lucene.apache.org/solr/guide/6_6/getting-started-with-solrcloud.html)
## Schemas
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/config/solr.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/config/solr.properties.j2 b/metron-platform/metron-solr/src/main/config/solr.properties.j2
index 00ad9dc..170aef3 100644
--- a/metron-platform/metron-solr/src/main/config/solr.properties.j2
+++ b/metron-platform/metron-solr/src/main/config/solr.properties.j2
@@ -35,7 +35,7 @@ indexing.input.topic={{indexing_input_topic}}
indexing.error.topic={{indexing_error_topic}}
##### Indexing #####
-indexing.writer.class.name={{ra_indexing_writer_class_name}}
+indexing.writer.class.name=org.apache.metron.solr.writer.SolrWriter
##### Parallelism #####
kafka.spout.parallelism={{ra_indexing_kafka_spout_parallelism}}
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/create_collection.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/create_collection.sh b/metron-platform/metron-solr/src/main/scripts/create_collection.sh
new file mode 100755
index 0000000..37fbccb
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/scripts/create_collection.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/$METRON_VERSION
+SOLR_VERSION=${global_solr_version}
+SOLR_USER=solr
+SOLR_SERVICE=$SOLR_USER
+SOLR_VAR_DIR="/var/$SOLR_SERVICE"
+
+cd $SOLR_VAR_DIR/solr-${SOLR_VERSION}
+su $SOLR_USER -c "bin/solr create -c $1 -d $METRON_HOME/config/schema/$1/"
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
new file mode 100755
index 0000000..68f0c20
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/$METRON_VERSION
+SOLR_VERSION=${global_solr_version}
+SOLR_USER=solr
+SOLR_SERVICE=$SOLR_USER
+SOLR_VAR_DIR="/var/$SOLR_SERVICE"
+
+cd $SOLR_VAR_DIR/solr-${SOLR_VERSION}
+su $SOLR_USER -c "bin/solr delete -c $1"
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/install_solr.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/install_solr.sh b/metron-platform/metron-solr/src/main/scripts/install_solr.sh
index 73415c8..da04557 100755
--- a/metron-platform/metron-solr/src/main/scripts/install_solr.sh
+++ b/metron-platform/metron-solr/src/main/scripts/install_solr.sh
@@ -30,7 +30,7 @@
service kibana stop
service elasticsearch stop
-SOLR_VERSION=6.6.2
+SOLR_VERSION=${global_solr_version}
SOLR_USER=solr
SOLR_SERVICE=$SOLR_USER
SOLR_VAR_DIR="/var/$SOLR_SERVICE"
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/start_solr.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/start_solr.sh b/metron-platform/metron-solr/src/main/scripts/start_solr.sh
new file mode 100755
index 0000000..6a89508
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/scripts/start_solr.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/$METRON_VERSION
+SOLR_VERSION=${global_solr_version}
+SOLR_USER=solr
+SOLR_SERVICE=$SOLR_USER
+SOLR_VAR_DIR="/var/$SOLR_SERVICE"
+
+cd $SOLR_VAR_DIR/solr-${SOLR_VERSION}
+su $SOLR_USER -c "bin/solr -e cloud -noprompt"
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/metron-platform/metron-solr/src/main/scripts/stop_solr.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/stop_solr.sh b/metron-platform/metron-solr/src/main/scripts/stop_solr.sh
new file mode 100755
index 0000000..5fe8d9c
--- /dev/null
+++ b/metron-platform/metron-solr/src/main/scripts/stop_solr.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+METRON_VERSION=${project.version}
+METRON_HOME=/usr/metron/$METRON_VERSION
+SOLR_VERSION=${global_solr_version}
+SOLR_USER=solr
+SOLR_SERVICE=$SOLR_USER
+SOLR_VAR_DIR="/var/$SOLR_SERVICE"
+
+cd $SOLR_VAR_DIR/solr-${SOLR_VERSION}
+su $SOLR_USER -c "bin/solr stop -all"
http://git-wip-us.apache.org/repos/asf/metron/blob/84cac7e8/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3ee242c..38738b5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -107,7 +107,7 @@
<global_slf4j_version>1.7.7</global_slf4j_version>
<global_opencsv_version>3.7</global_opencsv_version>
<global_java_version>1.8</global_java_version>
- <global_solr_version>6.6.0</global_solr_version>
+ <global_solr_version>6.6.2</global_solr_version>
<global_mockito_version>1.10.19</global_mockito_version>
<global_shade_version>2.4.3</global_shade_version>
<global_jackson_version>2.7.4</global_jackson_version>