You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by rl...@apache.org on 2017/12/01 21:40:46 UTC
[10/29] ambari git commit: AMBARI-22485 : Allow Ambari to support
non-kerberos SASL mechanisms for Kafka (ydavis via mradhakrishnan)
AMBARI-22485 : Allow Ambari to support non-kerberos SASL mechanisms for Kafka (ydavis via mradhakrishnan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d3429491
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d3429491
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d3429491
Branch: refs/heads/branch-feature-AMBARI-20859
Commit: d3429491a231f921ca3960ac0363c52690fb9dfa
Parents: 63a1587
Author: Madhuvanthi Radhakrishnan <mr...@hortonworks.com>
Authored: Thu Nov 30 11:24:44 2017 -0800
Committer: Madhuvanthi Radhakrishnan <mr...@hortonworks.com>
Committed: Thu Nov 30 11:24:44 2017 -0800
----------------------------------------------------------------------
.../libraries/functions/constants.py | 1 +
.../KAFKA/0.8.1/configuration/kafka-env.xml | 2 +-
.../KAFKA/0.8.1/package/scripts/kafka.py | 4 +-
.../KAFKA/0.8.1/package/scripts/params.py | 16 +++++--
.../0.8.1/package/scripts/setup_ranger_kafka.py | 8 ++--
.../KAFKA/0.8.1/package/scripts/upgrade.py | 3 +-
.../package/templates/kafka_client_jaas.conf.j2 | 3 ++
.../0.8.1/package/templates/kafka_jaas.conf.j2 | 47 ++++++++++++++++++++
.../KAFKA/0.9.0/configuration/kafka-broker.xml | 2 +-
.../HDP/2.0.6/properties/stack_features.json | 5 +++
10 files changed, 77 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index f46b7cf..b811861 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -120,3 +120,4 @@ class StackFeature:
RANGER_KMS_SSL = "ranger_kms_ssl"
KAFKA_ACL_MIGRATION_SUPPORT = "kafka_acl_migration_support"
ATLAS_CORE_SITE_SUPPORT="atlas_core_site_support"
+ KAFKA_EXTENDED_SASL_SUPPORT = "kafka_extended_sasl_support"
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml
index 7607c22..ea6c50c 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/configuration/kafka-env.xml
@@ -94,7 +94,7 @@ export JAVA_HOME={{java64_home}}
export PATH=$PATH:$JAVA_HOME/bin
export PID_DIR={{kafka_pid_dir}}
export LOG_DIR={{kafka_log_dir}}
-{% if security_enabled %}
+{% if kerberos_security_enabled or kafka_other_sasl_enabled %}
export KAFKA_KERBEROS_PARAMS="-Djavax.security.auth.useSubjectCredsOnly=false {{kafka_kerberos_params}}"
{% else %}
export KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py
index 3aa3473..3adea69 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/kafka.py
@@ -53,7 +53,7 @@ def kafka(upgrade_type=None):
Logger.info(format("Kafka listeners: {listeners}"))
kafka_server_config['listeners'] = listeners
- if params.security_enabled and params.kafka_kerberos_enabled:
+ if params.kerberos_security_enabled and params.kafka_kerberos_enabled:
Logger.info("Kafka kerberos security is enabled.")
kafka_server_config['advertised.listeners'] = listeners
Logger.info(format("Kafka advertised listeners: {listeners}"))
@@ -115,7 +115,7 @@ def kafka(upgrade_type=None):
content=InlineTemplate(params.log4j_props)
)
- if params.security_enabled and params.kafka_kerberos_enabled:
+ if (params.kerberos_security_enabled and params.kafka_kerberos_enabled) or params.kafka_other_sasl_enabled:
if params.kafka_jaas_conf_template:
File(format("{conf_dir}/kafka_jaas.conf"),
owner=params.kafka_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
index a62265b..46fdfba 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/params.py
@@ -157,13 +157,19 @@ if has_metric_collector:
pass
# Security-related params
-security_enabled = config['configurations']['cluster-env']['security_enabled']
+kerberos_security_enabled = config['configurations']['cluster-env']['security_enabled']
+
kafka_kerberos_enabled = (('security.inter.broker.protocol' in config['configurations']['kafka-broker']) and
((config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "PLAINTEXTSASL") or
(config['configurations']['kafka-broker']['security.inter.broker.protocol'] == "SASL_PLAINTEXT")))
+kafka_other_sasl_enabled = not kerberos_security_enabled and check_stack_feature(StackFeature.KAFKA_LISTENERS, stack_version_formatted) and \
+ check_stack_feature(StackFeature.KAFKA_EXTENDED_SASL_SUPPORT, stack_version_formatted) and \
+ (("SASL_PLAINTEXT" in config['configurations']['kafka-broker']['listeners']) or
+ ("PLAINTEXTSASL" in config['configurations']['kafka-broker']['listeners']) or
+ ("SASL_SSL" in config['configurations']['kafka-broker']['listeners']))
-if security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env'] \
+if kerberos_security_enabled and stack_version_formatted != "" and 'kafka_principal_name' in config['configurations']['kafka-env'] \
and check_stack_feature(StackFeature.KAFKA_KERBEROS, stack_version_formatted):
_hostname_lowercase = config['hostname'].lower()
_kafka_principal_name = config['configurations']['kafka-env']['kafka_principal_name']
@@ -171,6 +177,8 @@ if security_enabled and stack_version_formatted != "" and 'kafka_principal_name'
kafka_keytab_path = config['configurations']['kafka-env']['kafka_keytab']
kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name)
kafka_kerberos_params = "-Djava.security.auth.login.config="+ conf_dir +"/kafka_jaas.conf"
+elif kafka_other_sasl_enabled:
+ kafka_kerberos_params = "-Djava.security.auth.login.config="+ conf_dir +"/kafka_jaas.conf"
else:
kafka_kerberos_params = ''
kafka_jaas_principal = None
@@ -266,7 +274,7 @@ if enable_ranger_kafka and is_supported_kafka_ranger:
if len(custom_ranger_service_config) > 0:
ranger_plugin_config.update(custom_ranger_service_config)
- if stack_supports_ranger_kerberos and security_enabled:
+ if stack_supports_ranger_kerberos and kerberos_security_enabled:
ranger_plugin_config['policy.download.auth.users'] = kafka_user
ranger_plugin_config['tag.download.auth.users'] = kafka_user
ranger_plugin_config['ambari.service.check.user'] = policy_user
@@ -327,7 +335,7 @@ HdfsResource = functools.partial(
HdfsResource,
user=hdfs_user,
hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
- security_enabled = security_enabled,
+ security_enabled = kerberos_security_enabled,
keytab = hdfs_user_keytab,
kinit_path_local = kinit_path_local,
hadoop_bin_dir = hadoop_bin_dir,
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
index e9719aa..9aa09df 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/setup_ranger_kafka.py
@@ -66,10 +66,10 @@ def setup_ranger_kafka():
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble,
- is_security_enabled = params.security_enabled,
+ is_security_enabled = params.kerberos_security_enabled,
is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
- component_user_principal=params.kafka_jaas_principal if params.security_enabled else None,
- component_user_keytab=params.kafka_keytab_path if params.security_enabled else None)
+ component_user_principal=params.kafka_jaas_principal if params.kerberos_security_enabled else None,
+ component_user_keytab=params.kafka_keytab_path if params.kerberos_security_enabled else None)
if params.enable_ranger_kafka:
Execute(('cp', '--remove-destination', params.setup_ranger_env_sh_source, params.setup_ranger_env_sh_target),
@@ -81,7 +81,7 @@ def setup_ranger_kafka():
group = params.user_group,
mode = 0755
)
- if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.has_namenode and params.security_enabled:
+ if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.has_namenode and params.kerberos_security_enabled:
Logger.info("Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions")
setup_core_site_for_required_plugins(component_user=params.kafka_user,component_group=params.user_group,create_core_site_path = params.conf_dir, config = params.config)
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py
index e79a8ad..b327e54 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/scripts/upgrade.py
@@ -1,4 +1,3 @@
-
#!/usr/bin/env python
"""
Licensed to the Apache Software Foundation (ASF) under one
@@ -43,7 +42,7 @@ def run_migration(env, upgrade_type):
if params.upgrade_direction is None:
raise Fail('Parameter "upgrade_direction" is missing.')
- if not params.security_enabled:
+ if not params.kerberos_security_enabled:
Logger.info("Skip running the Kafka ACL migration script since cluster security is not enabled.")
return
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2
index 7f81d85..873d030 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_client_jaas.conf.j2
@@ -15,12 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
+
+{% if kerberos_security_enabled %}
KafkaClient {
com.sun.security.auth.module.Krb5LoginModule required
useTicketCache=true
renewTicket=true
serviceName="{{kafka_bare_jaas_principal}}";
};
+{% endif %}
Client {
com.sun.security.auth.module.Krb5LoginModule required
useTicketCache=true
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2
index 1d9e61d..68b5e44 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/kafka_jaas.conf.j2
@@ -15,6 +15,51 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
+
+/**
+* Example of SASL/PLAIN Configuration
+*
+* KafkaServer {
+* org.apache.kafka.common.security.plain.PlainLoginModule required
+* username="admin"
+* password="admin-secret"
+* user_admin="admin-secret"
+* user_alice="alice-secret";
+* };
+*
+* Example of SASL/SCRAM
+*
+* KafkaServer {
+* org.apache.kafka.common.security.scram.ScramLoginModule required
+* username="admin"
+* password="admin-secret"
+* };
+*
+* Example of Enabling multiple SASL mechanisms in a broker:
+*
+* KafkaServer {
+*
+* com.sun.security.auth.module.Krb5LoginModule required
+* useKeyTab=true
+* storeKey=true
+* keyTab="/etc/security/keytabs/kafka_server.keytab"
+* principal="kafka/kafka1.hostname.com@EXAMPLE.COM";
+*
+* org.apache.kafka.common.security.plain.PlainLoginModule required
+* username="admin"
+* password="admin-secret"
+* user_admin="admin-secret"
+* user_alice="alice-secret";
+*
+* org.apache.kafka.common.security.scram.ScramLoginModule required
+* username="scram-admin"
+* password="scram-admin-secret";
+* };
+*
+**/
+
+{% if kerberos_security_enabled %}
+
KafkaServer {
com.sun.security.auth.module.Krb5LoginModule required
useKeyTab=true
@@ -50,3 +95,5 @@ com.sun.security.jgss.krb5.initiate {
serviceName="{{kafka_bare_jaas_principal}}"
principal="{{kafka_jaas_principal}}";
};
+
+{% endif %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml b/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml
index 4cd2b0d..fbc1c64 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.9.0/configuration/kafka-broker.xml
@@ -17,7 +17,7 @@
* limitations under the License.
*/
-->
-<configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true">
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
<property>
<name>listeners</name>
<value>PLAINTEXT://localhost:6667</value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d3429491/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index f79cfe0..2109a5d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -447,6 +447,11 @@
"name": "sam_storage_core_in_registry",
"description": "Storage core module moved to registry",
"min_version": "2.6.3.0"
+ },
+ {
+ "name": "kafka_extended_sasl_support",
+ "description": "Support SASL PLAIN and GSSAPI",
+ "min_version": "2.6.5.0"
}
]
}