You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/09/07 16:53:06 UTC
[1/2] ambari git commit: AMBARI-18321. atlas hook for hive and storm
fail to push metadeta (dgrinenko via dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-2.4 05d56deb3 -> 535039422
refs/heads/trunk 60d96b155 -> 6e60b469c
AMBARI-18321. atlas hook for hive and storm fail to push metadeta (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6e60b469
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6e60b469
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6e60b469
Branch: refs/heads/trunk
Commit: 6e60b469c5ff87a328f0849d08c21c0730d0649e
Parents: 60d96b1
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Wed Sep 7 19:51:12 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Wed Sep 7 19:51:12 2016 +0300
----------------------------------------------------------------------
.../libraries/script/script.py | 2 +-
.../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 24 +++++++++++-
.../package/scripts/metadata_server.py | 18 ++++++++-
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 40 +++++++++++++++++++
.../0.1.0.2.3/package/scripts/status_params.py | 1 +
.../package/templates/atlas_kafka_acl.sh.j2 | 9 +++++
.../package/templates/kafka_jaas.conf.j2 | 41 ++++++++++++++++++++
.../ATLAS/0.7.0.2.5/kerberos.json | 3 ++
.../package/scripts/hbase_regionserver.py | 6 ---
.../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 6 +++
.../src/test/python/TestAmbariServer.py | 15 +++++--
.../src/test/python/stacks/utils/RMFTestCase.py | 15 +++----
12 files changed, 160 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 33cdcfc..6e567d2 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -326,7 +326,7 @@ class Script(object):
pids = []
for pid_file in pid_files:
if not sudo.path_exists(pid_file):
- raise Fail("Pid file {0} doesn't exist after starting of the component.")
+ raise Fail("Pid file {0} doesn't exist after starting of the component.".format(pid_file))
pids.append(sudo.read_file(pid_file).strip())
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 6ec636c..b16f597 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -17,13 +17,15 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-
+from resource_management import Package
+from resource_management import StackFeature
from resource_management.core.resources.system import Directory, File
from resource_management.core.source import StaticFile, InlineTemplate, Template
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.decorator import retry
from resource_management.libraries.functions import solr_cloud_util
+from resource_management.libraries.functions.stack_features import check_stack_feature, get_stack_feature_version
from resource_management.libraries.resources.properties_file import PropertiesFile
from resource_management.libraries.resources.template_config import TemplateConfig
@@ -128,6 +130,26 @@ def metadata(type='server'):
content=Template("atlas_hbase_setup.rb.j2")
)
+ if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) and\
+ params.security_enabled and not params.host_with_kafka:
+
+ File(params.atlas_kafka_setup,
+ group=params.user_group,
+ owner=params.kafka_user,
+ content=Template("atlas_kafka_acl.sh.j2")
+ )
+
+ File(format("{kafka_conf_dir}/kafka-env.sh"),
+ owner=params.kafka_user,
+ content=InlineTemplate(params.kafka_env_sh_template)
+ )
+
+ File(format("{kafka_conf_dir}/kafka_jaas.conf"),
+ group=params.user_group,
+ owner=params.kafka_user,
+ content=Template("kafka_jaas.conf.j2")
+ )
+
def upload_conf_set(config_set, jaasFile):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
index 3a93c80..a469ebb 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
@@ -17,6 +17,7 @@ limitations under the License.
"""
from metadata import metadata
+from resource_management import Fail
from resource_management.libraries.functions import conf_select, stack_select
from resource_management.core.resources.system import Execute, File
from resource_management.libraries.script.script import Script
@@ -27,7 +28,7 @@ from resource_management.libraries.functions.security_commons import build_expec
get_params_from_filesystem, validate_security_config_properties, \
FILE_TYPE_PROPERTIES
from resource_management.libraries.functions.show_logs import show_logs
-from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import check_stack_feature, get_stack_feature_version
from resource_management.libraries.functions.constants import StackFeature
from resource_management.core.resources.system import Directory
from resource_management.core.logger import Logger
@@ -71,7 +72,10 @@ class MetadataServer(Script):
daemon_cmd = format('source {params.conf_dir}/atlas-env.sh ; {params.metadata_start_script}')
no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1')
atlas_hbase_setup_command = format("cat {atlas_hbase_setup} | hbase shell -n")
+ atlas_kafka_setup_command = format("bash {atlas_kafka_setup}")
secure_atlas_hbase_setup_command = format("kinit -kt {hbase_user_keytab} {hbase_principal_name}; ") + atlas_hbase_setup_command
+ # in case if principal was distributed across several hosts, pattern need to be replaced to right one
+ secure_atlas_kafka_setup_command = format("kinit -kt {kafka_keytab} {kafka_principal_name}; ").replace("_HOST", params.hostname) + atlas_kafka_setup_command
if params.stack_supports_atlas_ranger_plugin:
Logger.info('Atlas plugin is enabled, configuring Atlas plugin.')
@@ -80,7 +84,7 @@ class MetadataServer(Script):
Logger.info('Atlas plugin is not supported or enabled.')
try:
- effective_version = format_stack_version(params.version) if upgrade_type is not None else params.stack_version_formatted
+ effective_version = get_stack_feature_version(params.config)
if check_stack_feature(StackFeature.ATLAS_HBASE_SETUP, effective_version):
if params.security_enabled and params.has_hbase_master:
@@ -96,6 +100,16 @@ class MetadataServer(Script):
user=params.hbase_user
)
+ if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, effective_version) and params.security_enabled:
+ try:
+ Execute(secure_atlas_kafka_setup_command,
+ user=params.kafka_user,
+ tries=5,
+ try_sleep=10
+ )
+ except Fail:
+ pass # do nothing and do not block Atlas start, fail logs would be available via Execute internals
+
Execute(daemon_cmd,
user=params.metadata_user,
not_if=no_op_test
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 3c1a81c..464de02 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -19,6 +19,9 @@ limitations under the License.
"""
import os
import sys
+
+from ambari_commons import OSCheck
+from resource_management import get_bare_principal
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.format import format
@@ -233,6 +236,7 @@ ranger_admin_hosts = default('/clusterHostInfo/ranger_admin_hosts', [])
has_ranger_admin = not len(ranger_admin_hosts) == 0
atlas_hbase_setup = format("{exec_tmp_dir}/atlas_hbase_setup.rb")
+atlas_kafka_setup = format("{exec_tmp_dir}/atlas_kafka_acl.sh")
atlas_graph_storage_hbase_table = default('/configurations/application-properties/atlas.graph.storage.hbase.table', None)
atlas_audit_hbase_tablename = default('/configurations/application-properties/atlas.audit.hbase.tablename', None)
@@ -240,6 +244,42 @@ hbase_user_keytab = default('/configurations/hbase-env/hbase_user_keytab', None)
hbase_principal_name = default('/configurations/hbase-env/hbase_principal_name', None)
enable_ranger_hbase = False
+# ToDo: Kafka port to Atlas
+# Used while upgrading the stack in a kerberized cluster and running kafka-acls.sh
+hosts_with_kafka = default('/clusterHostInfo/kafka_broker_hosts', [])
+host_with_kafka = hostname in hosts_with_kafka
+
+ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", [])
+has_ranger_tagsync = len(ranger_tagsync_hosts) > 0
+ranger_user = default('/configurations/ranger-env/ranger_user', None)
+
+kafka_keytab = default('/configurations/kafka-env/kafka_keytab', None)
+kafka_principal_name = default('/configurations/kafka-env/kafka_principal_name', None)
+
+if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, version_for_stack_feature_checks):
+ default_replication_factor = default('/configurations/application-properties/atlas.notification.replicas', None)
+
+ kafka_env_sh_template = config['configurations']['kafka-env']['content']
+ kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+ kafka_conf_dir = os.path.join(kafka_home, "config")
+
+ kafka_zk_endpoint = default("/configurations/kafka-broker/zookeeper.connect", None)
+ kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker']) and
+ ((config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "PLAINTEXTSASL") or
+ (config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "SASL_PLAINTEXT")))
+ if security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env'] \
+ and check_stack_feature(StackFeature.KAFKA_KERBEROS, stack_version_formatted):
+ _hostname_lowercase = config['hostname'].lower()
+ _kafka_principal_name = config['configurations']['kafka-env']['kafka_principal_name']
+ kafka_jaas_principal = _kafka_principal_name.replace('_HOST', _hostname_lowercase)
+ kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab']
+ kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name)
+ kafka_kerberos_params = "-Djava.security.auth.login.config={0}/kafka_jaas.conf".format(kafka_conf_dir)
+ else:
+ kafka_kerberos_params = ''
+ kafka_jaas_principal = None
+ kafka_keytab_path = None
+
if has_ranger_admin and stack_supports_atlas_ranger_plugin:
# for create_hdfs_directory
namenode_host = set(default("/clusterHostInfo/namenode_host", []))
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
index 59bcbaf..1fd1b9b 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
@@ -46,6 +46,7 @@ pid_file = format("{pid_dir}/atlas.pid")
metadata_user = default("/configurations/atlas-env/metadata_user", None)
hbase_user = default("/configurations/hbase-env/hbase_user", None)
+kafka_user = default("/configurations/kafka-env/kafka_user", None)
# Security related/required params
hostname = config['hostname']
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2 b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
new file mode 100644
index 0000000..9e77ac4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
@@ -0,0 +1,9 @@
+#!/usr/bin/bash
+
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}} --add --topic ATLAS_HOOK --allow-principal User:* --producer
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}} --add --topic ATLAS_HOOK --allow-principal User:{{metadata_user}} --consumer --group atlas
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}} --add --topic ATLAS_ENTITIES --allow-principal User:{{metadata_user}} --producer
+
+{% if has_ranger_tagsync %}
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}} --add --topic ATLAS_ENTITIES --allow-principal User:{{ranger_user}} --consumer --group ranger_entities_consumer
+{% endif %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
new file mode 100644
index 0000000..56c558d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
@@ -0,0 +1,41 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+KafkaServer {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{kafka_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="{{kafka_bare_jaas_principal}}"
+ principal="{{kafka_jaas_principal}}";
+};
+KafkaClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useTicketCache=true
+ renewTicket=true
+ serviceName="{{kafka_bare_jaas_principal}}";
+};
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{kafka_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="zookeeper"
+ principal="{{kafka_jaas_principal}}";
+};
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
index a9da650..bc8e351 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
@@ -84,6 +84,9 @@
"keytab": {
"configuration": "ranger-atlas-audit/xasecure.audit.jaas.Client.option.keyTab"
}
+ },
+ {
+ "name": "/KAFKA/KAFKA_BROKER/kafka_broker"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
index 9b1257d..226e7fd5 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
@@ -87,10 +87,6 @@ class HbaseRegionServerDefault(HbaseRegionServer):
env.set_params(params)
upgrade.post_regionserver(env)
- def post_start(self, env, upgrade_type=None):
- import params
-
-
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
@@ -99,8 +95,6 @@ class HbaseRegionServerDefault(HbaseRegionServer):
hbase_service('regionserver', action='start')
- self.post_start(env, upgrade_type=upgrade_type)
-
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
index 4f92cf9..72d612a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
@@ -69,6 +69,9 @@
<name>ambari-infra-solr-client</name>
<condition>should_install_infra_solr_client</condition>
</package>
+ <package>
+ <name>kafka_${stack_version}</name>
+ </package>
</packages>
</osSpecific>
<osSpecific>
@@ -81,6 +84,9 @@
<name>ambari-infra-solr-client</name>
<condition>should_install_infra_solr_client</condition>
</package>
+ <package>
+ <name>kafka-${stack_version}</name>
+ </package>
</packages>
</osSpecific>
</osSpecifics>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 37af554..9992eb3 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -56,7 +56,18 @@ shutil.copyfile(project_dir+"/ambari-server/conf/unix/ambari.properties", "/tmp/
# We have to use this import HACK because the filename contains a dash
_search_file = os_utils.search_file
-os_utils.search_file = MagicMock(return_value="/tmp/ambari.properties")
+
+
+def search_file_proxy(filename, searchpatch, pathsep=os.pathsep):
+ global _search_file
+
+ if "ambari.properties" in filename:
+ return "/tmp/ambari.properties"
+
+ return _search_file(filename, searchpatch, pathsep)
+
+
+os_utils.search_file = search_file_proxy
with patch.object(platform, "linux_distribution", return_value = MagicMock(return_value=('Redhat', '6.4', 'Final'))):
with patch("os.path.isdir", return_value = MagicMock(return_value=True)):
with patch("os.access", return_value = MagicMock(return_value=True)):
@@ -65,7 +76,6 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
with patch("os.symlink"):
with patch("glob.glob", return_value = ['/etc/init.d/postgresql-9.3']):
_ambari_server_ = __import__('ambari-server')
- os_utils.search_file = _search_file
with patch("__builtin__.open"):
from ambari_commons.firewall import Firewall
from ambari_commons.os_check import OSCheck, OSConst
@@ -119,7 +129,6 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
from ambari_server.hostUpdate import update_host_names
from ambari_server.checkDatabase import check_database
from ambari_server import serverConfiguration
- serverConfiguration.search_file = _search_file
CURR_AMBARI_VERSION = "2.0.0"
http://git-wip-us.apache.org/repos/asf/ambari/blob/6e60b469/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
index 9240d38..a101577 100644
--- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
+++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
@@ -146,14 +146,15 @@ class RMFTestCase(TestCase):
with patch('resource_management.core.shell.call', side_effect=call_mocks) as mocks_dict['call']:
with patch.object(Script, 'get_config', return_value=self.config_dict) as mocks_dict['get_config']: # mocking configurations
with patch.object(Script, 'get_tmp_dir', return_value="/tmp") as mocks_dict['get_tmp_dir']:
- with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local) as mocks_dict['get_kinit_path']:
- with patch.object(platform, 'linux_distribution', return_value=os_type) as mocks_dict['linux_distribution']:
- with patch.object(os, "environ", new=os_env) as mocks_dict['environ']:
- if not try_install:
- with patch.object(Script, 'install_packages') as install_mock_value:
+ with patch.object(Script, 'post_start') as mocks_dict['post_start']:
+ with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local) as mocks_dict['get_kinit_path']:
+ with patch.object(platform, 'linux_distribution', return_value=os_type) as mocks_dict['linux_distribution']:
+ with patch.object(os, "environ", new=os_env) as mocks_dict['environ']:
+ if not try_install:
+ with patch.object(Script, 'install_packages') as install_mock_value:
+ method(RMFTestCase.env, *command_args)
+ else:
method(RMFTestCase.env, *command_args)
- else:
- method(RMFTestCase.env, *command_args)
sys.path.remove(scriptsdir)
[2/2] ambari git commit: AMBARI-18321. atlas hook for hive and storm
fail to push metadeta (dgrinenko via dlysnichenko)
Posted by dm...@apache.org.
AMBARI-18321. atlas hook for hive and storm fail to push metadeta (dgrinenko via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/53503942
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/53503942
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/53503942
Branch: refs/heads/branch-2.4
Commit: 5350394220ce6a84c351f0afa633d3a0cf7e179e
Parents: 05d56de
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Wed Sep 7 19:52:33 2016 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Wed Sep 7 19:52:33 2016 +0300
----------------------------------------------------------------------
.../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 24 +++++++++++-
.../package/scripts/metadata_server.py | 18 ++++++++-
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 38 ++++++++++++++++++
.../0.1.0.2.3/package/scripts/status_params.py | 1 +
.../package/templates/atlas_kafka_acl.sh.j2 | 9 +++++
.../package/templates/kafka_jaas.conf.j2 | 41 ++++++++++++++++++++
.../ATLAS/0.7.0.2.5/kerberos.json | 3 ++
.../package/scripts/hbase_regionserver.py | 6 ---
.../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 6 +++
.../src/test/python/TestAmbariServer.py | 15 +++++--
10 files changed, 149 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 6ec636c..b16f597 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -17,13 +17,15 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-
+from resource_management import Package
+from resource_management import StackFeature
from resource_management.core.resources.system import Directory, File
from resource_management.core.source import StaticFile, InlineTemplate, Template
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.decorator import retry
from resource_management.libraries.functions import solr_cloud_util
+from resource_management.libraries.functions.stack_features import check_stack_feature, get_stack_feature_version
from resource_management.libraries.resources.properties_file import PropertiesFile
from resource_management.libraries.resources.template_config import TemplateConfig
@@ -128,6 +130,26 @@ def metadata(type='server'):
content=Template("atlas_hbase_setup.rb.j2")
)
+ if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, get_stack_feature_version(params.config)) and\
+ params.security_enabled and not params.host_with_kafka:
+
+ File(params.atlas_kafka_setup,
+ group=params.user_group,
+ owner=params.kafka_user,
+ content=Template("atlas_kafka_acl.sh.j2")
+ )
+
+ File(format("{kafka_conf_dir}/kafka-env.sh"),
+ owner=params.kafka_user,
+ content=InlineTemplate(params.kafka_env_sh_template)
+ )
+
+ File(format("{kafka_conf_dir}/kafka_jaas.conf"),
+ group=params.user_group,
+ owner=params.kafka_user,
+ content=Template("kafka_jaas.conf.j2")
+ )
+
def upload_conf_set(config_set, jaasFile):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
index ed0314b..712c15c 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata_server.py
@@ -17,6 +17,7 @@ limitations under the License.
"""
from metadata import metadata
+from resource_management import Fail
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
from resource_management import Execute, File, check_process_status, Script, format_stack_version
@@ -25,7 +26,7 @@ from resource_management.libraries.functions.security_commons import build_expec
get_params_from_filesystem, validate_security_config_properties, \
FILE_TYPE_PROPERTIES
from resource_management.libraries.functions.show_logs import show_logs
-from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.stack_features import check_stack_feature, get_stack_feature_version
from resource_management.libraries.functions import StackFeature
from resource_management.core.resources.system import Directory
from resource_management.core.logger import Logger
@@ -69,7 +70,10 @@ class MetadataServer(Script):
daemon_cmd = format('source {params.conf_dir}/atlas-env.sh ; {params.metadata_start_script}')
no_op_test = format('ls {params.pid_file} >/dev/null 2>&1 && ps -p `cat {params.pid_file}` >/dev/null 2>&1')
atlas_hbase_setup_command = format("cat {atlas_hbase_setup} | hbase shell -n")
+ atlas_kafka_setup_command = format("bash {atlas_kafka_setup}")
secure_atlas_hbase_setup_command = format("kinit -kt {hbase_user_keytab} {hbase_principal_name}; ") + atlas_hbase_setup_command
+ # in case if principal was distributed across several hosts, pattern need to be replaced to right one
+ secure_atlas_kafka_setup_command = format("kinit -kt {kafka_keytab} {kafka_principal_name}; ").replace("_HOST", params.hostname) + atlas_kafka_setup_command
if params.stack_supports_atlas_ranger_plugin:
Logger.info('Atlas plugin is enabled, configuring Atlas plugin.')
@@ -78,7 +82,7 @@ class MetadataServer(Script):
Logger.info('Atlas plugin is not supported or enabled.')
try:
- effective_version = format_stack_version(params.version) if upgrade_type is not None else params.stack_version_formatted
+ effective_version = get_stack_feature_version(params.config)
if check_stack_feature(StackFeature.ATLAS_HBASE_SETUP, effective_version):
if params.security_enabled and params.has_hbase_master:
@@ -94,6 +98,16 @@ class MetadataServer(Script):
user=params.hbase_user
)
+ if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, effective_version) and params.security_enabled:
+ try:
+ Execute(secure_atlas_kafka_setup_command,
+ user=params.kafka_user,
+ tries=5,
+ try_sleep=10
+ )
+ except Fail:
+ pass # do nothing and do not block Atlas start, fail logs would be available via Execute internals
+
Execute(daemon_cmd,
user=params.metadata_user,
not_if=no_op_test
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 028a828..45e323c 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -22,6 +22,7 @@ import os
import sys
# Local Imports
+from resource_management import get_bare_principal
from status_params import *
from resource_management import format_stack_version, Script
from resource_management.libraries.functions import format
@@ -233,6 +234,7 @@ ranger_admin_hosts = default('/clusterHostInfo/ranger_admin_hosts', [])
has_ranger_admin = not len(ranger_admin_hosts) == 0
atlas_hbase_setup = format("{exec_tmp_dir}/atlas_hbase_setup.rb")
+atlas_kafka_setup = format("{exec_tmp_dir}/atlas_kafka_acl.sh")
atlas_graph_storage_hbase_table = default('/configurations/application-properties/atlas.graph.storage.hbase.table', None)
atlas_audit_hbase_tablename = default('/configurations/application-properties/atlas.audit.hbase.tablename', None)
@@ -240,6 +242,42 @@ hbase_user_keytab = default('/configurations/hbase-env/hbase_user_keytab', None)
hbase_principal_name = default('/configurations/hbase-env/hbase_principal_name', None)
enable_ranger_hbase = False
+# ToDo: Kafka port to Atlas
+# Used while upgrading the stack in a kerberized cluster and running kafka-acls.sh
+hosts_with_kafka = default('/clusterHostInfo/kafka_broker_hosts', [])
+host_with_kafka = hostname in hosts_with_kafka
+
+ranger_tagsync_hosts = default("/clusterHostInfo/ranger_tagsync_hosts", [])
+has_ranger_tagsync = len(ranger_tagsync_hosts) > 0
+ranger_user = default('/configurations/ranger-env/ranger_user', None)
+
+kafka_keytab = default('/configurations/kafka-env/kafka_keytab', None)
+kafka_principal_name = default('/configurations/kafka-env/kafka_principal_name', None)
+
+if check_stack_feature(StackFeature.ATLAS_UPGRADE_SUPPORT, version_for_stack_feature_checks):
+ default_replication_factor = default('/configurations/application-properties/atlas.notification.replicas', None)
+
+ kafka_env_sh_template = config['configurations']['kafka-env']['content']
+ kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+ kafka_conf_dir = os.path.join(kafka_home, "config")
+
+ kafka_zk_endpoint = default("/configurations/kafka-broker/zookeeper.connect", None)
+ kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker']) and
+ ((config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "PLAINTEXTSASL") or
+ (config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "SASL_PLAINTEXT")))
+ if security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env'] \
+ and check_stack_feature(StackFeature.KAFKA_KERBEROS, stack_version_formatted):
+ _hostname_lowercase = config['hostname'].lower()
+ _kafka_principal_name = config['configurations']['kafka-env']['kafka_principal_name']
+ kafka_jaas_principal = _kafka_principal_name.replace('_HOST', _hostname_lowercase)
+ kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab']
+ kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name)
+ kafka_kerberos_params = "-Djava.security.auth.login.config={0}/kafka_jaas.conf".format(kafka_conf_dir)
+ else:
+ kafka_kerberos_params = ''
+ kafka_jaas_principal = None
+ kafka_keytab_path = None
+
if has_ranger_admin and stack_supports_atlas_ranger_plugin:
# for create_hdfs_directory
namenode_host = set(default("/clusterHostInfo/namenode_host", []))
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
index 0f0d89c..3cf81ca 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/status_params.py
@@ -46,6 +46,7 @@ pid_file = format("{pid_dir}/atlas.pid")
metadata_user = default("/configurations/atlas-env/metadata_user", None)
hbase_user = default("/configurations/hbase-env/hbase_user", None)
+kafka_user = default("/configurations/kafka-env/kafka_user", None)
# Security related/required params
hostname = config['hostname']
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2 b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
new file mode 100644
index 0000000..9e77ac4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/atlas_kafka_acl.sh.j2
@@ -0,0 +1,9 @@
+#!/usr/bin/bash
+
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}} --add --topic ATLAS_HOOK --allow-principal User:* --producer
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}} --add --topic ATLAS_HOOK --allow-principal User:{{metadata_user}} --consumer --group atlas
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}} --add --topic ATLAS_ENTITIES --allow-principal User:{{metadata_user}} --producer
+
+{% if has_ranger_tagsync %}
+{{kafka_home}}/bin/kafka-acls.sh --authorizer-properties zookeeper.connect={{kafka_zk_endpoint}} --add --topic ATLAS_ENTITIES --allow-principal User:{{ranger_user}} --consumer --group ranger_entities_consumer
+{% endif %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
new file mode 100644
index 0000000..56c558d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/templates/kafka_jaas.conf.j2
@@ -0,0 +1,41 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+KafkaServer {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{kafka_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="{{kafka_bare_jaas_principal}}"
+ principal="{{kafka_jaas_principal}}";
+};
+KafkaClient {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useTicketCache=true
+ renewTicket=true
+ serviceName="{{kafka_bare_jaas_principal}}";
+};
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ keyTab="{{kafka_keytab_path}}"
+ storeKey=true
+ useTicketCache=false
+ serviceName="zookeeper"
+ principal="{{kafka_jaas_principal}}";
+};
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
index a9da650..bc8e351 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.7.0.2.5/kerberos.json
@@ -84,6 +84,9 @@
"keytab": {
"configuration": "ranger-atlas-audit/xasecure.audit.jaas.Client.option.keyTab"
}
+ },
+ {
+ "name": "/KAFKA/KAFKA_BROKER/kafka_broker"
}
]
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
index cf65718..370167b 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_regionserver.py
@@ -83,10 +83,6 @@ class HbaseRegionServerDefault(HbaseRegionServer):
env.set_params(params)
upgrade.post_regionserver(env)
- def post_start(self, env, upgrade_type=None):
- import params
-
-
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
@@ -95,8 +91,6 @@ class HbaseRegionServerDefault(HbaseRegionServer):
hbase_service('regionserver', action='start')
- self.post_start(env, upgrade_type=upgrade_type)
-
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
index 4f92cf9..72d612a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
@@ -69,6 +69,9 @@
<name>ambari-infra-solr-client</name>
<condition>should_install_infra_solr_client</condition>
</package>
+ <package>
+ <name>kafka_${stack_version}</name>
+ </package>
</packages>
</osSpecific>
<osSpecific>
@@ -81,6 +84,9 @@
<name>ambari-infra-solr-client</name>
<condition>should_install_infra_solr_client</condition>
</package>
+ <package>
+ <name>kafka-${stack_version}</name>
+ </package>
</packages>
</osSpecific>
</osSpecifics>
http://git-wip-us.apache.org/repos/asf/ambari/blob/53503942/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index a45a4bd..66b85ea 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -53,7 +53,18 @@ shutil.copyfile(project_dir+"/ambari-server/conf/unix/ambari.properties", "/tmp/
# We have to use this import HACK because the filename contains a dash
_search_file = os_utils.search_file
-os_utils.search_file = MagicMock(return_value="/tmp/ambari.properties")
+
+
+def search_file_proxy(filename, searchpatch, pathsep=os.pathsep):
+ global _search_file
+
+ if "ambari.properties" in filename:
+ return "/tmp/ambari.properties"
+
+ return _search_file(filename, searchpatch, pathsep)
+
+
+os_utils.search_file = search_file_proxy
with patch.object(platform, "linux_distribution", return_value = MagicMock(return_value=('Redhat', '6.4', 'Final'))):
with patch("os.path.isdir", return_value = MagicMock(return_value=True)):
with patch("os.access", return_value = MagicMock(return_value=True)):
@@ -62,7 +73,6 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
with patch("os.symlink"):
with patch("glob.glob", return_value = ['/etc/init.d/postgresql-9.3']):
_ambari_server_ = __import__('ambari-server')
- os_utils.search_file = _search_file
with patch("__builtin__.open"):
from ambari_commons.firewall import Firewall
from ambari_commons.os_check import OSCheck, OSConst
@@ -116,7 +126,6 @@ with patch.object(platform, "linux_distribution", return_value = MagicMock(retur
from ambari_server.hostUpdate import update_host_names
from ambari_server.checkDatabase import check_database
from ambari_server import serverConfiguration
- serverConfiguration.search_file = _search_file
CURR_AMBARI_VERSION = "2.0.0"