You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2019/05/09 17:55:31 UTC
[metron] branch feature/METRON-2088-support-hdp-3.1 updated:
METRON-2097 Install Metron MPack in Ambari 2.7.3.0 (nickwallen) closes
apache/metron#1397
This is an automated email from the ASF dual-hosted git repository.
nickallen pushed a commit to branch feature/METRON-2088-support-hdp-3.1
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/feature/METRON-2088-support-hdp-3.1 by this push:
new 1245fd1 METRON-2097 Install Metron MPack in Ambari 2.7.3.0 (nickwallen) closes apache/metron#1397
1245fd1 is described below
commit 1245fd160a5ae826a8e2900528ab12c64350fdc6
Author: nickwallen <ni...@nickallen.org>
AuthorDate: Thu May 9 13:54:59 2019 -0400
METRON-2097 Install Metron MPack in Ambari 2.7.3.0 (nickwallen) closes apache/metron#1397
---
.../ambari_config/tasks/dependencies-CentOS-7.yml | 14 +++------
.../roles/ambari_gather_facts/tasks/main.yml | 36 ++++++++++------------
.../roles/sensor-stubs/templates/start-bro-stub | 7 +++--
.../roles/sensor-stubs/templates/start-snort-stub | 7 +++--
.../roles/sensor-stubs/templates/start-yaf-stub | 7 +++--
.../centos7/ansible/inventory/group_vars/all | 6 +++-
.../5.6.14/configuration/elastic-env.xml | 10 +++---
.../ELASTICSEARCH/5.6.14/package/scripts/params.py | 22 +++++++++++--
.../KIBANA/5.6.14/configuration/kibana-env.xml | 12 +++++---
.../KIBANA/5.6.14/package/scripts/params.py | 21 +++++++++++--
.../src/main/resources/mpack.json | 11 +++++--
.../METRON/CURRENT/configuration/metron-env.xml | 16 +++++-----
.../CURRENT/package/scripts/params/params.py | 22 +++++++++++--
.../CURRENT/package/scripts/params/params_linux.py | 20 +++++-------
.../package/scripts/params/status_params.py | 14 ++++++---
.../METRON/CURRENT/themes/metron_theme.json | 1 +
.../metron-mpack/src/main/resources/mpack.json | 16 +++++++---
17 files changed, 155 insertions(+), 87 deletions(-)
diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml
index 72a96d4..d7fc829 100644
--- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml
@@ -15,12 +15,8 @@
# limitations under the License.
#
---
-- name: Install urllib3 with pip
- pip:
- name: urllib3
- version: 1.10.2
-
-- name: Install requests with pip
- pip:
- name: requests
- version: 2.6.1
+- name: Install Ambari dependencies
+ yum: name={{ item }}
+ with_items:
+ - python-urllib3
+ - python-requests
diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
index 25f0982..c76c7bf 100644
--- a/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
@@ -56,8 +56,8 @@
#
- name: "Ask Ambari: core_site_tag"
shell: >
- {{ curl }} '{{ base_url }}/hosts/{{ namenode_host }}/host_components/NAMENODE' \
- | python -c '{{ parse_json }}["HostRoles"]["actual_configs"]["core-site"]["default"]'
+ {{ curl }} '{{ base_url }}/configurations?type=core-site' \
+ | python -c '{{ parse_json }}["items"][-1]["tag"]'
args:
warn: false
register: core_site_tag_response
@@ -104,17 +104,16 @@
# kafka_broker_tag
#
- name: "Ask Ambari: kafka_broker_tag"
- uri:
- url: "{{ base_url }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
+ shell: >
+ {{ curl }} '{{ base_url }}/configurations?type=kafka-broker' \
+ | python -c '{{ parse_json }}["items"][-1]["tag"]'
+ args:
+ warn: false
register: kafka_broker_tag_response
when: kafka_broker_tag is undefined
- set_fact:
- kafka_broker_tag: "{{ (kafka_broker_tag_response.content | from_json).HostRoles.actual_configs['kafka-broker'].default }}"
+ kafka_broker_tag: "{{ kafka_broker_tag_response.stdout_lines[0] }}"
when: kafka_broker_tag is undefined
#
@@ -158,17 +157,16 @@
# zookeeper_tag
#
- name: "Ask Ambari: zookeeper_tag"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ zookeeper_hosts[0] }}/host_components/ZOOKEEPER_SERVER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
+ shell: >
+ {{ curl }} '{{ base_url }}/configurations?type=zookeeper-env' \
+ | python -c '{{ parse_json }}["items"][-1]["tag"]'
+ args:
+ warn: false
register: zookeeper_tag_response
when: zookeeper_tag is undefined
- set_fact:
- zookeeper_tag: "{{ (zookeeper_tag_response.content | from_json).HostRoles.actual_configs['zoo.cfg'].default }}"
+ zookeeper_tag: "{{ zookeeper_tag_response.stdout_lines[0] }}"
when: zookeeper_tag is undefined
#
@@ -176,7 +174,7 @@
#
- name: "Ask Ambari: zookeeper_url, zookeeper_port"
uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}"
+ url: "{{ base_url }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -197,7 +195,7 @@
#
- name: "Ask Ambari: metron_hosts"
uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/METRON/components/METRON_INDEXING"
+ url: "{{ base_url }}/services/METRON/components/METRON_INDEXING"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -214,7 +212,7 @@
#
- name: "Ask Ambari: kibana_hosts"
uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KIBANA/components/KIBANA_MASTER"
+ url: "{{ base_url }}/services/KIBANA/components/KIBANA_MASTER"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
index 24027b3..f45f78e 100644
--- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
@@ -58,11 +58,12 @@ if [ ${SECURITY_ENABLED,,} == 'true' ]; then
fi
while true; do
-
# transform the bro timestamp and push to kafka
SEARCH="\"ts\"\:[0-9]\+\."
REPLACE="\"ts\"\:`date +%s`\."
- shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list $BROKERLIST --topic $TOPIC --security-protocol $KAFKA_SECURITY_PROTOCOL
-
+ shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER \
+ --broker-list $BROKERLIST \
+ --topic $TOPIC \
+ --producer-property security.protocol=$KAFKA_SECURITY_PROTOCOL
sleep $DELAY
done
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
index c60c002..b37bafc 100644
--- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
@@ -58,11 +58,12 @@ if [ ${SECURITY_ENABLED,,} == 'true' ]; then
fi
while true; do
-
# transform the timestamp and push to kafka
SEARCH="[^,]\+ ,"
REPLACE="`date +'%m\/%d\/%y-%H:%M:%S'`.000000 ,"
- shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list $BROKERLIST --topic $TOPIC --security-protocol $KAFKA_SECURITY_PROTOCOL
-
+ shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER \
+ --broker-list $BROKERLIST \
+ --topic $TOPIC \
+ --producer-property security.protocol=$KAFKA_SECURITY_PROTOCOL
sleep $DELAY
done
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub
index c218c60..22b3fbe 100644
--- a/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub
@@ -58,11 +58,12 @@ if [ ${SECURITY_ENABLED,,} == 'true' ]; then
fi
while true; do
-
# transform the timestamp and push to kafka
SEARCH="[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\} [0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}\.[0-9]\+"
REPLACE="`date +'%Y-%m-%d %H:%M:%S'`.000"
- shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list $BROKERLIST --topic $TOPIC --security-protocol $KAFKA_SECURITY_PROTOCOL
-
+ shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER \
+ --broker-list $BROKERLIST \
+ --topic $TOPIC \
+ --producer-property security.protocol=$KAFKA_SECURITY_PROTOCOL
sleep $DELAY
done
diff --git a/metron-deployment/development/centos7/ansible/inventory/group_vars/all b/metron-deployment/development/centos7/ansible/inventory/group_vars/all
index ba37eed..a88e7c8 100644
--- a/metron-deployment/development/centos7/ansible/inventory/group_vars/all
+++ b/metron-deployment/development/centos7/ansible/inventory/group_vars/all
@@ -18,7 +18,11 @@
# only need to build the RPMs for CentOS
metron_build_packages_cmd: "shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -Pbuild-rpms"
+# Ambari 2.7.3.0
+centos7_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.7.3.0/ambari.repo
+
cluster_type: single_node_vm
elasticsearch_hosts: "{{ groups.search | join(',') }}"
hdp_host_group: "{{ groups.ambari_slave }}"
-hdp_stack: "2.6"
+hdp_stack: "3.1"
+
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14/configuration/elastic-env.xml
index 9e4f8ad..ca8a56d 100755
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14/configuration/elastic-env.xml
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14/configuration/elastic-env.xml
@@ -27,6 +27,12 @@
<value-attributes>
<type>user</type>
<overridable>false</overridable>
+ <user-groups>
+ <property>
+ <type>elastic-env</type>
+ <name>elastic_group</name>
+ </property>
+ </user-groups>
</value-attributes>
</property>
<property>
@@ -34,10 +40,6 @@
<value>elasticsearch</value>
<property-type>GROUP</property-type>
<description>Service group for Elasticsearch</description>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
</property>
<property>
<name>elastic_log_dir</name>
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14/package/scripts/params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14/package/scripts/params.py
index 24f2306..92060cd 100755
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14/package/scripts/params.py
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.14/package/scripts/params.py
@@ -26,6 +26,23 @@ def yamlify_variables(var) :
else:
return var
+
+def get_java_home(config):
+ if 'ambariLevelParams' in config.keys():
+ # Ambari 2.7.x
+ return config['ambariLevelParams']['java_home']
+ else:
+ # Ambari 2.6.x
+ return config['hostLevelParams']['java_home']
+
+def get_hostname(config):
+ if 'agentLevelParams' in config.keys():
+ # Ambari 2.7.x
+ return config['agentLevelParams']['hostname']
+ else:
+ # Ambari 2.6.x
+ return config['hostname']
+
# server configurations
config = Script.get_config()
@@ -43,8 +60,9 @@ elastic_group = config['configurations']['elastic-env']['elastic_group']
log_dir = config['configurations']['elastic-env']['elastic_log_dir']
pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
+hostname = get_hostname(config)
+java64_home = get_java_home(config)
+
elastic_env_sh_template = config['configurations']['elastic-env']['content']
sysconfig_template = config['configurations']['elastic-sysconfig']['content']
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.14/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.14/configuration/kibana-env.xml
index 1246405..cb1ec7f 100755
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.14/configuration/kibana-env.xml
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.14/configuration/kibana-env.xml
@@ -27,17 +27,19 @@
<value-attributes>
<type>user</type>
<overridable>false</overridable>
+ <user-groups>
+ <property>
+ <type>kibana-env</type>
+ <name>kibana_group</name>
+ </property>
+ </user-groups>
</value-attributes>
</property>
<property>
- <name>kabana_group</name>
+ <name>kibana_group</name>
<value>kibana</value>
<property-type>GROUP</property-type>
<description>Service Group for Kibana</description>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
</property>
<property require-input="true">
<name>kibana_server_host</name>
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.14/package/scripts/params.py b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.14/package/scripts/params.py
index ef4cb62..c60f54d 100755
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.14/package/scripts/params.py
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/common-services/KIBANA/5.6.14/package/scripts/params.py
@@ -25,6 +25,22 @@ from urlparse import urlparse
from resource_management.libraries.functions import format
from resource_management.libraries.script import Script
+def get_java_home(config):
+ if 'ambariLevelParams' in config.keys():
+ # Ambari 2.7.x
+ return config['ambariLevelParams']['java_home']
+ else:
+ # Ambari 2.6.x
+ return config['hostLevelParams']['java_home']
+
+def get_hostname(config):
+ if 'agentLevelParams' in config.keys():
+ # Ambari 2.7.x
+ return config['agentLevelParams']['hostname']
+ else:
+ # Ambari 2.6.x
+ return config['hostname']
+
# server configurations
config = Script.get_config()
@@ -44,7 +60,6 @@ es_port = parsed.netloc.split(':')[1]
kibana_port = config['configurations']['kibana-env']['kibana_server_port']
kibana_server_host = config['configurations']['kibana-env']['kibana_server_host']
kibana_default_application = config['configurations']['kibana-env']['kibana_default_application']
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
+hostname = get_hostname(config)
+java64_home = get_java_home(config)
kibana_yml_template = config['configurations']['kibana-site']['content']
-
diff --git a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
index 43c4d26..40ffb68 100644
--- a/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
+++ b/metron-deployment/packaging/ambari/elasticsearch-mpack/src/main/resources/mpack.json
@@ -42,8 +42,11 @@
{
"stack_name" : "HDP",
"stack_version" : "2.6"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "3.1"
}
-
]
},
{
@@ -65,9 +68,11 @@
{
"stack_name" : "HDP",
"stack_version" : "2.6"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "3.1"
}
-
-
]
}
]
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index e644b31..eaf7b0c 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -43,8 +43,14 @@
<description>Service User for Metron</description>
<display-name>Metron User Name</display-name>
<value-attributes>
- <type>user</type>
- <overridable>false</overridable>
+ <type>user</type>
+ <overridable>false</overridable>
+ <user-groups>
+ <property>
+ <type>metron-env</type>
+ <name>metron_group</name>
+ </property>
+ </user-groups>
</value-attributes>
</property>
<property>
@@ -52,11 +58,7 @@
<value>metron</value>
<property-type>GROUP</property-type>
<description>Service Group for Metron</description>
- <display-name>Metron Group Name</display-name>
- <value-attributes>
- <type>user</type>
- <overridable>false</overridable>
- </value-attributes>
+ <display-name>Metron Service Group</display-name>
</property>
<property>
<name>metron_log_dir</name>
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params.py
index 953435d..fba9e9c 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params.py
@@ -26,7 +26,23 @@ if OSCheck.is_windows_family():
else:
from params_linux import *
-java_home = config['hostLevelParams']['java_home']
-java_version = expect("/hostLevelParams/java_version", int)
+def get_java_home(config):
+ if 'ambariLevelParams' in config.keys():
+ # Ambari 2.7.x
+ return config['ambariLevelParams']['java_home']
+ else:
+ # Ambari 2.6.x
+ return config['hostLevelParams']['java_home']
-host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
+def get_java_version(config):
+ if 'ambariLevelParams' in config.keys():
+ # Ambari 2.7.x
+ return expect("/ambariLevelParams/java_version", int)
+ else:
+ # Ambari 2.6.x
+ return expect("/hostLevelParams/java_version", int)
+
+java_home = get_java_home(config)
+java_version = get_java_version(config)
+
+host_sys_prepped = default("/ambariLevelParams/host_sys_prepped", False)
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 64105e3..746cd6d 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -41,7 +41,7 @@ tmp_dir = Script.get_tmp_dir()
hdp_version = default("/commandParams/version", None)
-hostname = config['hostname']
+hostname = status_params.hostname
metron_home = status_params.metron_home
metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
@@ -130,17 +130,11 @@ kafka_home = os.path.join(stack_root, "current", "kafka-broker")
kafka_bin_dir = os.path.join(kafka_home, "bin")
# zookeeper
-zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
-has_zk_host = not len(zk_hosts) == 0
-zookeeper_quorum = None
-if has_zk_host:
- if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
- zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
- else:
- zookeeper_clientPort = '2181'
- zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
- # last port config
- zookeeper_quorum += ':' + zookeeper_clientPort
+zookeeper_port = default('/configurations/zoo.cfg/clientPort', '2181')
+zookeeper_hosts_property = 'zookeeper_hosts' if 'zookeeper_hosts' in config['clusterHostInfo'] else "zookeeper_server_hosts"
+zookeeper_hosts = config['clusterHostInfo'][zookeeper_hosts_property]
+zookeeper_host_ports = map(lambda host: host + ":" + str(zookeeper_port), zookeeper_hosts)
+zookeeper_quorum = ",".join(zookeeper_host_ports)
# Solr params
solr_version = '6.6.2'
@@ -189,7 +183,7 @@ smokeuser_principal = config['configurations']['cluster-env']['smokeuser_princip
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
-dfs_type = default("/commandParams/dfs_type", "")
+dfs_type = default("/clusterLevelParams/dfs_type", "")
# create partial functions with common arguments for every HdfsResource call
# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index fc5daac..694c7fa 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -23,9 +23,17 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions import default, format
from resource_management.libraries.functions.version import format_stack_version
+def get_hostname(config):
+ if 'agentLevelParams' in config.keys():
+ # Ambari 2.7.x
+ return config['agentLevelParams']['hostname']
+ else:
+ # Ambari 2.6.x
+ return config['hostname']
+
config = Script.get_config()
-hostname = config['hostname']
+hostname = get_hostname(config)
metron_user = config['configurations']['metron-env']['metron_user']
metron_home = config['configurations']['metron-env']['metron_home']
metron_zookeeper_config_dir = config['configurations']['metron-env']['metron_zookeeper_config_dir']
@@ -117,10 +125,6 @@ storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
zeppelin_server_url = config['configurations']['metron-env']['zeppelin_server_url']
zeppelin_shiro_ini_content = config['configurations']['zeppelin-shiro-ini']['shiro_ini_content']
-# Security
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-stack_version_formatted = format_stack_version(stack_version_unformatted)
-
security_enabled = config['configurations']['cluster-env']['security_enabled']
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
tmp_dir = Script.get_tmp_dir()
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 69084e3..02b8fb0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -1,4 +1,5 @@
{
+ "name": "default",
"configuration": {
"layouts": [
{
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
index d314ebf..36f9068 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
@@ -42,8 +42,11 @@
{
"stack_name" : "HDP",
"stack_version" : "2.6"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "3.1"
}
-
]
},
{
@@ -65,9 +68,11 @@
{
"stack_name" : "HDP",
"stack_version" : "2.6"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "3.1"
}
-
-
]
},
{
@@ -89,8 +94,11 @@
{
"stack_name" : "HDP",
"stack_version" : "2.6"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "3.1"
}
-
]
}
]