You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2020/03/15 15:17:20 UTC
[bigtop] branch master updated: BIGTOP-3300. Add puppet manifests
for hadoop-kms. (#615)
This is an automated email from the ASF dual-hosted git repository.
evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 6307ef3 BIGTOP-3300. Add puppet manifests for hadoop-kms. (#615)
6307ef3 is described below
commit 6307ef3b99e250d786b128bd6039f7f5701dccbf
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Mon Mar 16 00:17:10 2020 +0900
BIGTOP-3300. Add puppet manifests for hadoop-kms. (#615)
* BIGTOP-3300. Add puppet manifests for hadoop-kms.
* fixed role assingment, kms kerberos configs and bugs in kerberos module.
* tightened permission of kms-env.sh containing keystore password.
---
bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml | 19 ++-
bigtop-deploy/puppet/manifests/cluster.pp | 3 +
.../puppet/modules/hadoop/manifests/init.pp | 71 ++++++++
.../puppet/modules/hadoop/templates/core-site.xml | 7 +
.../puppet/modules/hadoop/templates/kms-env.sh | 100 ++++++++++++
.../puppet/modules/hadoop/templates/kms-site.xml | 181 +++++++++++++++++++++
.../puppet/modules/kerberos/manifests/init.pp | 9 +-
7 files changed, 380 insertions(+), 10 deletions(-)
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 9e7ce80..d73b929 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -69,15 +69,15 @@
# Kerberos
#hadoop::hadoop_security_authentication: "kerberos"
-#kerberos::site::domain: "do.main"
-#kerberos::site::realm: "DO.MAIN"
-#kerberos::site::kdc_server: "localhost"
-#kerberos::site::kdc_port: "88"
-#kerberos::site::admin_port: "749"
-#kerberos::site::keytab_export_dir: "/var/lib/bigtop_keytabs"
+#kerberos::krb_site::domain: "bigtop.apache.org"
+#kerberos::krb_site::realm: "BIGTOP.APACHE.ORG"
+#kerberos::krb_site::kdc_server: "%{hiera('bigtop::hadoop_head_node')}"
+#kerberos::krb_site::kdc_port: "88"
+#kerberos::krb_site::admin_port: "749"
+#kerberos::krb_site::keytab_export_dir: "/var/lib/bigtop_keytabs"
-# applies to hdfs, yarn, mapred and httpfs
-hadoop::kerberos_realm: "%{hiera('kerberos::site::realm')}"
+# applies to hdfs, yarn, mapred, kms and httpfs
+hadoop::kerberos_realm: "%{hiera('kerberos::krb_site::realm')}"
hadoop::common_hdfs::hadoop_namenode_host: "%{hiera('bigtop::hadoop_head_node')}"
# actually default but needed for hadoop_namenode_uri here
@@ -121,6 +121,9 @@ hadoop::common_yarn::yarn_log_server_url: "%{hiera('bigtop::hadoop_history_serve
hadoop::httpfs::hadoop_httpfs_port: "14000"
+hadoop::kms_host: "%{hiera('bigtop::hadoop_head_node')}"
+hadoop::kms_port: "16000"
+
bigtop::hadoop_zookeeper_port: "2181"
hadoop::zk: "%{hiera('bigtop::hadoop_head_node')}:%{hiera('bigtop::hadoop_zookeeper_port')}"
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index f9b3564..95ced2d 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -57,6 +57,9 @@ $roles_map = {
mapred => {
library => ["mapred-app"],
},
+ kms => {
+ master => ["kms"],
+ },
hbase => {
master => ["hbase-master"],
worker => ["hbase-server"],
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 9d83924..4c3947e 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -23,6 +23,8 @@ class hadoop ($hadoop_security_authentication = "simple",
hive => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,users', hosts => "*" },
httpfs => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,hive,httpfs,users', hosts => "*" } },
$generate_secrets = false,
+ $kms_host = undef,
+ $kms_port = undef,
) {
include stdlib
@@ -74,6 +76,10 @@ class hadoop ($hadoop_security_authentication = "simple",
include hadoop::httpfs
}
+ if ("kms" in $roles) {
+ include hadoop::kms
+ }
+
if ("hadoop-client" in $roles) {
include hadoop::client
}
@@ -235,6 +241,8 @@ class hadoop ($hadoop_security_authentication = "simple",
$hadoop_http_authentication_cookie_domain = regsubst($fqdn, "^[^\\.]+\\.", ""),
$generate_secrets = $hadoop::generate_secrets,
$namenode_datanode_registration_ip_hostname_check = undef,
+ $kms_host = $hadoop::kms_host,
+ $kms_port = $hadoop::kms_port,
) inherits hadoop {
$sshfence_keydir = "$hadoop_ha_sshfence_user_home/.ssh"
@@ -359,6 +367,7 @@ class hadoop ($hadoop_security_authentication = "simple",
Kerberos::Host_keytab["HTTP"] -> Service<| title == "hadoop-hdfs-datanode" |>
Kerberos::Host_keytab["HTTP"] -> Service<| title == "hadoop-yarn-resourcemanager" |>
Kerberos::Host_keytab["HTTP"] -> Service<| title == "hadoop-yarn-nodemanager" |>
+ Kerberos::Host_keytab["HTTP"] -> Service<| title == "hadoop-kms" |>
}
}
@@ -512,6 +521,67 @@ class hadoop ($hadoop_security_authentication = "simple",
Kerberos::Host_keytab <| title == "httpfs" |> -> Service["hadoop-httpfs"]
}
+ class kms ($kms_host = $hadoop::kms_host,
+ $kms_port = $hadoop::kms_port,
+ $secret = "hadoop kms secret",
+ $generate_secrets = $hadoop::generate_secrets,
+ $hadoop_core_proxyusers = $hadoop::proxyusers,
+ $hadoop_security_authentcation = $hadoop::hadoop_security_authentication,
+ $kerberos_realm = $hadoop::kerberos_realm,
+ ) inherits hadoop {
+ include hadoop::common_hdfs
+
+ if ($hadoop_security_authentication == "kerberos") {
+ kerberos::host_keytab { "kms":
+ spnego => true,
+ require => Package["hadoop-kms"],
+ }
+ }
+
+ package { "hadoop-kms":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ file { "/etc/hadoop-kms/conf/kms-site.xml":
+ content => template('hadoop/kms-site.xml'),
+ require => [Package["hadoop-kms"]],
+ }
+
+ file { "/etc/hadoop-kms/conf/kms-env.sh":
+ content => template('hadoop/kms-env.sh'),
+ owner => 'kms',
+ group => 'kms',
+ mode => '0400',
+ require => [Package["hadoop-kms"]],
+ }
+
+ if $generate_secrets {
+ $kms_signature_secret = trocla("kms-signature-secret", "plain")
+ } else {
+ $kms_signature_secret = $secret
+ }
+ if $kms_signature_secret == undef {
+ fail("KMS signature secret must be set")
+ }
+
+ file { "/etc/hadoop-kms/conf/kms-signature.secret":
+ content => $kms_signature_secret,
+ # it's a password file - do not filebucket
+ backup => false,
+ require => [Package["hadoop-kms"]],
+ }
+
+ service { "hadoop-kms":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-kms"], File["/etc/hadoop-kms/conf/kms-site.xml"], File["/etc/hadoop-kms/conf/kms-env.sh"], File["/etc/hadoop-kms/conf/kms-signature.secret"],
+ File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"]],
+ require => [ Package["hadoop-kms"] ],
+ }
+ Kerberos::Host_keytab <| title == "kms" |> -> Service["hadoop-kms"]
+ }
+
class kinit {
include hadoop::common_hdfs
@@ -893,4 +963,5 @@ class hadoop ($hadoop_security_authentication = "simple",
require => [Package["jdk"], Package["hadoop"], Package["hadoop-hdfs"], Package["hadoop-mapreduce"]],
}
}
+
}
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
index f4aab69..72f0bc2 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
@@ -169,4 +169,11 @@
<value>com.quantcast.qfs.hadoop.QuantcastFileSystem</value>
</property>
<% end -%>
+
+<% if @kms_host %>
+ <property>
+ <name>hadoop.security.key.provider.path</name>
+ <value>kms://http@<%= @kms_host %>:<%= @kms_port %>/kms</value>
+ </property>
+<% end -%>
</configuration>
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/kms-env.sh b/bigtop-deploy/puppet/modules/hadoop/templates/kms-env.sh
new file mode 100644
index 0000000..736ee74
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/kms-env.sh
@@ -0,0 +1,100 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# Set kms specific environment variables here.
+
+# Settings for the Embedded Tomcat that runs KMS
+# Java System properties for KMS should be specified in this variable
+#
+# export CATALINA_OPTS=
+
+# KMS logs directory
+#
+# export KMS_LOG=${KMS_HOME}/logs
+
+# KMS temporary directory
+#
+# export KMS_TEMP=${KMS_HOME}/temp
+
+# The HTTP port used by KMS
+#
+export KMS_HTTP_PORT=<%= @kms_port %>
+
+# The Admin port used by KMS
+#
+# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1`
+
+# The Tomcat protocol to use for handling requests.
+# The default HTTP/1.1 handler is thread-per-request.
+# The NIO handler multiplexes multiple requests per thread.
+#
+# export KMS_PROTOCOL="HTTP/1.1"
+# export KMS_PROTOCOL="org.apache.coyote.http11.Http11NioProtocol"
+
+# The maximum number of Tomcat handler threads
+#
+# export KMS_MAX_THREADS=1000
+
+# The maximum queue length for incoming connection requests when all possible
+# request processing threads are in use. Any requests received when the queue
+# is full will be refused.
+#
+# export KMS_ACCEPT_COUNT=500
+
+# The number of threads to be used to accept connections. Increase this value
+# on a multi CPU machine, although you would never really need more than 2.
+# Also, with a lot of non keep alive connections, you might want to increase
+# this value as well.
+#
+# Increasing this has no effect unless using the NIO protocol.
+#
+# export KMS_ACCEPTOR_THREAD_COUNT=1
+
+# The maximum size of Tomcat HTTP header
+#
+# export KMS_MAX_HTTP_HEADER_SIZE=65536
+
+# Set to 'true' if you want the SSL stack to require a valid certificate chain
+# from the client before accepting a connection. Set to 'want' if you want the
+# SSL stack to request a client Certificate, but not fail if one isn't
+# presented. A 'false' value (which is the default) will not require a
+# certificate chain unless the client requests a resource protected by a
+# security constraint that uses CLIENT-CERT authentication.
+#
+# export KMS_SSL_CLIENT_AUTH=false
+
+# The comma separated list of SSL protocols to support
+#
+# export KMS_SSL_ENABLED_PROTOCOLS="TLSv1,TLSv1.1,TLSv1.2,SSLv2Hello"
+
+# The comma separated list of encryption ciphers for SSL
+#
+# export KMS_SSL_CIPHERS=
+
+# The location of the SSL keystore if using SSL
+#
+# export KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
+
+# The password of the SSL keystore if using SSL
+#
+# export KMS_SSL_KEYSTORE_PASS=password
+
+# The full path to any native libraries that need to be loaded
+# (For eg. location of natively compiled tomcat Apache portable
+# runtime (APR) libraries
+#
+# export JAVA_LIBRARY_PATH=${HOME}/lib/native
+
+export HADOOP_KEYSTORE_PASSWORD="<%= @secret %>"
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/kms-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/kms-site.xml
new file mode 100644
index 0000000..06684c3
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/kms-site.xml
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+
+ <!-- KMS Backend KeyProvider -->
+
+ <property>
+ <name>hadoop.kms.key.provider.uri</name>
+ <value>jceks://file@/${user.home}/kms.keystore</value>
+ <description>
+ URI of the backing KeyProvider for the KMS.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.security.keystore.java-keystore-provider.password-file</name>
+ <value>kms.keystore.password</value>
+ <description>
+ If using the JavaKeyStoreProvider, the file name for the keystore password.
+ </description>
+ </property>
+
+ <!-- KMS Cache -->
+
+ <property>
+ <name>hadoop.kms.cache.enable</name>
+ <value>true</value>
+ <description>
+ Whether the KMS will act as a cache for the backing KeyProvider.
+ When the cache is enabled, operations like getKeyVersion, getMetadata,
+ and getCurrentKey will sometimes return cached data without consulting
+ the backing KeyProvider. Cached values are flushed when keys are deleted
+ or modified.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.cache.timeout.ms</name>
+ <value>600000</value>
+ <description>
+ Expiry time for the KMS key version and key metadata cache, in
+ milliseconds. This affects getKeyVersion and getMetadata.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.current.key.cache.timeout.ms</name>
+ <value>30000</value>
+ <description>
+ Expiry time for the KMS current key cache, in milliseconds. This
+ affects getCurrentKey operations.
+ </description>
+ </property>
+
+ <!-- KMS Audit -->
+
+ <property>
+ <name>hadoop.kms.audit.aggregation.window.ms</name>
+ <value>10000</value>
+ <description>
+ Duplicate audit log events within the aggregation window (specified in
+ ms) are quashed to reduce log traffic. A single message for aggregated
+ events is printed at the end of the window, along with a count of the
+ number of aggregated events.
+ </description>
+ </property>
+
+ <!-- KMS Security -->
+
+ <property>
+ <name>hadoop.kms.authentication.type</name>
+ <value><%= @hadoop_security_authentication %></value>
+ </property>
+<% if @hadoop_security_authentication == "kerberos" -%>
+ <property>
+ <name>hadoop.kms.authentication.kerberos.keytab</name>
+ <value>/etc/kms.keytab</value>
+ </property>
+ <property>
+ <name>hadoop.kms.authentication.kerberos.principal</name>
+ <value>HTTP/<%= @fqdn %>@<%= @kerberos_realm %></value>
+ </property>
+ <property>
+ <name>hadoop.kms.authentication.kerberos.name.rules</name>
+ <value>DEFAULT</value>
+ </property>
+<% end -%>
+
+<% if @hadoop_core_proxyusers -%>
+<% @hadoop_core_proxyusers.sort.each do |superuser, data| -%>
+ <property>
+ <name>hadoop.kms.proxyuser.<%= superuser %>.hosts</name>
+ <value><%= data['hosts'] %></value>
+ </property>
+ <property>
+ <name>hadoop.kms.proxyuser.<%= superuser %>.groups</name>
+ <value><%= data['groups'] %></value>
+ </property>
+<% end -%>
+<% end -%>
+
+ <!-- Authentication cookie signature source -->
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider</name>
+ <value>random</value>
+ <description>
+ Indicates how the secret to sign the authentication cookies will be
+ stored. Options are 'random' (default), 'string' and 'zookeeper'.
+ If using a setup with multiple KMS instances, 'zookeeper' should be used.
+ </description>
+ </property>
+
+ <!-- Configuration for 'zookeeper' authentication cookie signature source -->
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.path</name>
+ <value>/hadoop-kms/hadoop-auth-signature-secret</value>
+ <description>
+ The Zookeeper ZNode path where the KMS instances will store and retrieve
+ the secret from.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string</name>
+ <value>#HOSTNAME#:#PORT#,...</value>
+ <description>
+ The Zookeeper connection string, a list of hostnames and port comma
+ separated.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type</name>
+ <value>none</value>
+ <description>
+ The Zookeeper authentication type, 'none' (default) or 'sasl' (Kerberos).
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab</name>
+ <value>/etc/hadoop/conf/kms.keytab</value>
+ <description>
+ The absolute path for the Kerberos keytab with the credentials to
+ connect to Zookeeper.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal</name>
+ <value>kms/#HOSTNAME#</value>
+ <description>
+ The Kerberos service principal used to connect to Zookeeper.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.audit.logger</name>
+ <value>org.apache.hadoop.crypto.key.kms.server.SimpleKMSAuditLogger</value>
+ <description>
+ The audit logger for KMS. It is a comma-separated list of KMSAuditLogger
+ class names. Default is the text-format SimpleKMSAuditLogger only.
+ If this is not configured, default will be used.
+ </description>
+ </property>
+
+</configuration>
diff --git a/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp b/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
index 8b2c603..d6642bd 100644
--- a/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
@@ -74,6 +74,8 @@ class kerberos {
}
class kdc inherits kerberos::krb_site {
+ Class['kerberos::kdc'] -> Class['hadoop_cluster_node']
+
package { $package_name_kdc:
ensure => installed,
}
@@ -173,6 +175,8 @@ class kerberos {
command => "kadmin -w secure -p kadmin/admin -q 'addprinc -randkey $principal'",
unless => "kadmin -w secure -p kadmin/admin -q listprincs | grep -q $principal",
require => Package[$kerberos::krb_site::package_name_client],
+ tries => 180,
+ try_sleep => 1,
}
->
exec { "xst.$title":
@@ -189,8 +193,9 @@ class kerberos {
$keytab = "/etc/$title.keytab"
$internal_princs = $spnego ? {
- /(true|enabled)/ => [ 'HTTP' ],
- default => [ ],
+ true => [ 'HTTP' ],
+ 'enabled' => [ 'HTTP' ],
+ default => [ ],
}
realize(Kerberos::Principal[$internal_princs])