You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2015/05/19 21:45:53 UTC
bigtop git commit: BIGTOP-1872: Improve HUE puppet recipes
Repository: bigtop
Updated Branches:
refs/heads/master 3ee4257de -> 86f8929f0
BIGTOP-1872: Improve HUE puppet recipes
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/86f8929f
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/86f8929f
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/86f8929f
Branch: refs/heads/master
Commit: 86f8929f0f9b0f3cf3a19eb77f27b8f4db7ffb22
Parents: 3ee4257
Author: Olaf Flebbe <of...@oflebbe.de>
Authored: Wed May 13 22:15:34 2015 +0200
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue May 19 21:23:28 2015 +0200
----------------------------------------------------------------------
.../puppet/modules/hue/manifests/init.pp | 28 +-
.../puppet/modules/hue/templates/hue.ini | 276 ++++++++-----------
2 files changed, 133 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/86f8929f/bigtop-deploy/puppet/modules/hue/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hue/manifests/init.pp b/bigtop-deploy/puppet/modules/hue/manifests/init.pp
index 9974cb0..4fd9c1c 100644
--- a/bigtop-deploy/puppet/modules/hue/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hue/manifests/init.pp
@@ -14,13 +14,22 @@
# limitations under the License.
class hue {
- class server($sqoop_url, $solr_url, $hbase_thrift_url,
- $webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_url, $rm_proxy_url, $history_server_url,
- $hue_host = "0.0.0.0", $hue_port = "8888", $default_fs = "hdfs://localhost:8020",
- $kerberos_realm = "", $hue_apps = "all") {
+ class server($sqoop_url = "http://localhost:12000/sqoop", $solr_url = "http://localhost:8983/solr/", $hbase_thrift_url = "",
+ $webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_proxy_url, $history_server_url,
+ $hive_host = "", $hive_port = "10000",
+ $rm_logical_name = undef, $rm_api_port = "8088", $app_blacklist = "impala, security",
+ $hue_host = "0.0.0.0", $hue_port = "8888", $hue_timezone = "America/Los_Angeles",
+ $default_fs = "hdfs://localhost:8020",
+ $kerberos_realm = "", $kerberos_principal = "", $huecert = undef, $huekey = undef,
+ $auth_backend = "desktop.auth.backend.AllowFirstUserDjangoBackend",
+ $ldap_url = undef, $ldap_cert = undef, $use_start_tls = "true",
+ $base_dn = undef , $bind_dn = undef, $bind_password = undef,
+ $user_name_attr = undef, $user_filter = undef,
+ $group_member_attr = undef, $group_filter = undef,
+ $hue_apps = "all" ) {
$hue_packages = $hue_apps ? {
- "all" => [ "hue" ], # The hue metapackage requires all apps
+ "all" => [ "hue", "hue-server" ], # The hue metapackage requires all apps
"none" => [ "hue-server" ],
default => concat(prefix($hue_apps, "hue-"), [ "hue-server" ])
}
@@ -29,7 +38,7 @@ class hue {
require kerberos::client
kerberos::host_keytab { "hue":
spnego => false,
- require => Package[$hue_packages],
+ require => Package["hue-server"],
}
}
@@ -44,12 +53,11 @@ class hue {
service { "hue":
ensure => running,
- require => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"] ],
- subscribe => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"] ],
+ require => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"]],
+ subscribe => [ Package[$hue_packages], File["/etc/hue/conf/hue.ini"]],
hasrestart => true,
hasstatus => true,
- }
+ }
Kerberos::Host_keytab <| title == "hue" |> -> Service["hue"]
-
}
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/86f8929f/bigtop-deploy/puppet/modules/hue/templates/hue.ini
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hue/templates/hue.ini b/bigtop-deploy/puppet/modules/hue/templates/hue.ini
index 283c332..fe1b472 100644
--- a/bigtop-deploy/puppet/modules/hue/templates/hue.ini
+++ b/bigtop-deploy/puppet/modules/hue/templates/hue.ini
@@ -1,18 +1,3 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
# Hue configuration file
# ===================================
#
@@ -31,21 +16,21 @@
[desktop]
- send_dbug_messages=1
-
- # To show database transactions, set database_logging to 1
- database_logging=0
+# send_dbug_messages=1#
+#
+# # To show database transactions, set database_logging to 1
+# database_logging=0
# Set this to a random string, the longer the better.
# This is used for secure hashing in the session store.
- secret_key=deaddeafdeaddeafdeaddeafdeaddeafdeaddeafdeaddeafdeaddeaf
+ secret_key=Geheim!
# Webserver listens on this address and port
http_host=<%= @hue_host %>
http_port=<%= @hue_port %>
# Time zone name
- time_zone=America/Los_Angeles
+ time_zone=<%= @hue_timezone %>
# Enable or disable Django debug mode.
django_debug_mode=false
@@ -63,8 +48,8 @@
## django_email_backend=django.core.mail.backends.smtp.EmailBackend
# Webserver runs as this user
- server_user=hue
- server_group=hue
+ ## server_user=hue
+ ## server_group=hue
# This should be the Hue admin and proxy user
## default_user=hue
@@ -80,10 +65,14 @@
## cherrypy_server_threads=10
# Filename of SSL Certificate
- ## ssl_certificate=
+ <%if @huecert %>
+ ssl_certificate=<%= @huecert %>
+ <% end -%>
# Filename of SSL RSA Private Key
- ## ssl_private_key=
+ <%if @huekey %>
+ ssl_private_key=<%= @huekey %>
+ <% end -%>
# List of allowed and disallowed ciphers in cipher list format.
# See http://www.openssl.org/docs/apps/ciphers.html for more information on cipher list format.
@@ -99,7 +88,7 @@
# Help improve Hue with anonymous usage analytics.
# Use Google Analytics to see how many times an application or specific section of an application is used, nothing more.
- ## collect_usage=true
+ collect_usage=false
# Support for HTTPS termination at the load-balancer level with SECURE_PROXY_SSL_HEADER.
## secure_proxy_ssl_header=false
@@ -115,7 +104,7 @@
# Comma separated list of apps to not load at server startup.
# e.g.: pig,zookeeper
- ## app_blacklist=
+ app_blacklist=<%= @app_blacklist %>
# The directory where to store the auditing logs. Auditing is disable if the value is empty.
# e.g. /var/log/hue/audit.log
@@ -156,10 +145,10 @@
# - libopenid.backend.OpenIDBackend
# - liboauth.backend.OAuthBackend
# (Support Twitter, Facebook, Google+ and Linkedin
- ## backend=desktop.auth.backend.AllowFirstUserDjangoBackend
+ backend=<%= @auth_backend %>
# The service to use when querying PAM.
- ## pam_service=login
+ #pam_service=login
# When using the desktop.auth.backend.RemoteUserDjangoBackend, this sets
# the normalized name of the header that contains the remote user.
@@ -171,9 +160,6 @@
# Defaults to HTTP_REMOTE_USER
## remote_user_header=HTTP_REMOTE_USER
- # Synchronize a users groups when they login
- ## sync_groups_on_login=false
-
# Ignore the case of usernames when searching for existing users.
# Only supported in remoteUserDjangoBackend.
## ignore_username_case=false
@@ -193,118 +179,105 @@
# -------------------------------------------------------------------
[[ldap]]
+<% if @ldap_url %>
# The search base for finding users and groups
- ## base_dn="DC=mycompany,DC=com"
+ base_dn="<%= @base_dn %>"
# URL of the LDAP server
- ## ldap_url=ldap://auth.mycompany.com
+ ldap_url=<%= @ldap_url %>
# A PEM-format file containing certificates for the CA's that
# Hue will trust for authentication over TLS.
# The certificate for the CA that signed the
# LDAP server certificate must be included among these certificates.
# See more here http://www.openldap.org/doc/admin24/tls.html.
- ## ldap_cert=
- ## use_start_tls=true
+<% if @ldap_cert -%>
+ ldap_cert=<%= @ldap_cert %>
+<% end -%>
+ use_start_tls=<%= @use_start_tls %>
# Distinguished name of the user to bind as -- not necessary if the LDAP server
# supports anonymous searches
- ## bind_dn="CN=ServiceAccount,DC=mycompany,DC=com"
+<% if @bind_dn -%>
+ # Distinguished name of the user to bind as -- not necessary if the LDAP server
+ # supports anonymous searches
+ bind_dn="<%= @bind_dn %>"
# Password of the bind user -- not necessary if the LDAP server supports
# anonymous searches
- ## bind_password=
+ bind_password=<%= @bind_password %>
+<% end -%>
+<% if @user_filter -%>
+ # Use search bind authentication.
+ search_bind_authentication=true
+<% else -%>
# Pattern for searching for usernames -- Use <username> for the parameter
# For use when using LdapBackend for Hue authentication
- ## ldap_username_pattern="uid=<username>,ou=People,dc=mycompany,dc=com"
+ # ldap_username_pattern="uid=<username>,ou=People,dc=mycompany,dc=com"
+
+ search_bind_authentication=false
+<% end -%>
+ # Execute this script to produce the bind user password. This will be used
+ # when `bind_password` is not set.
+ ## bind_password_script=
# Create users in Hue when they try to login with their LDAP credentials
# For use when using LdapBackend for Hue authentication
- ## create_users_on_login = true
+ create_users_on_login = true
+
+ # Synchronize a users groups when they login
+ ## sync_groups_on_login=false
# Ignore the case of usernames when searching for existing users in Hue.
- ## ignore_username_case=false
+ ignore_username_case=true
# Force usernames to lowercase when creating new users from LDAP.
## force_username_lowercase=false
- # Use search bind authentication.
- ## search_bind_authentication=true
-
# Choose which kind of subgrouping to use: nested or suboordinate (deprecated).
## subgroups=suboordinate
# Define the number of levels to search for nested members.
## nested_members_search_depth=10
- [[[users]]]
+ # Whether or not to follow referrals
+ ## follow_referrals=false
- # Base filter for searching for users
- ## user_filter="objectclass=*"
+ # Enable python-ldap debugging.
+ ## debug=false
- # The username attribute in the LDAP schema
- ## user_name_attr=sAMAccountName
+ # Sets the debug level within the underlying LDAP C lib.
+ ## debug_level=255
+ # Possible values for trace_level are 0 for no logging, 1 for only logging the method calls with arguments,
+ # 2 for logging the method calls with arguments and the complete results and 9 for also logging the traceback of method calls.
+ ## trace_level=0
+
+ [[[users]]]
+<% if @user_filter -%>
+ # Base filter for searching for users
+ user_filter="<%= @user_filter %>"
+<% end -%>
+<% if @user_name_attr -%>
+ # The username attribute in the LDAP schema
+ user_name_attr=<%= @user_name_attr %>
+<% end -%>
[[[groups]]]
# Base filter for searching for groups
- ## group_filter="objectclass=*"
+<% if @group_filter -%>
+ group_filter="objectclass=groupOfEntries"
+<% end -%>
# The group name attribute in the LDAP schema
## group_name_attr=cn
# The attribute of the group object which identifies the members of the group
- ## group_member_attr=members
-
- [[[ldap_servers]]]
-
- ## [[[[mycompany]]]]
-
- # The search base for finding users and groups
- ## base_dn="DC=mycompany,DC=com"
-
- # URL of the LDAP server
- ## ldap_url=ldap://auth.mycompany.com
-
- # A PEM-format file containing certificates for the CA's that
- # Hue will trust for authentication over TLS.
- # The certificate for the CA that signed the
- # LDAP server certificate must be included among these certificates.
- # See more here http://www.openldap.org/doc/admin24/tls.html.
- ## ldap_cert=
- ## use_start_tls=true
-
- # Distinguished name of the user to bind as -- not necessary if the LDAP server
- # supports anonymous searches
- ## bind_dn="CN=ServiceAccount,DC=mycompany,DC=com"
-
- # Password of the bind user -- not necessary if the LDAP server supports
- # anonymous searches
- ## bind_password=
-
- # Pattern for searching for usernames -- Use <username> for the parameter
- # For use when using LdapBackend for Hue authentication
- ## ldap_username_pattern="uid=<username>,ou=People,dc=mycompany,dc=com"
-
- ## Use search bind authentication.
- ## search_bind_authentication=true
-
- ## [[[[[users]]]]]
-
- # Base filter for searching for users
- ## user_filter="objectclass=Person"
-
- # The username attribute in the LDAP schema
- ## user_name_attr=sAMAccountName
-
- ## [[[[[groups]]]]]
-
- # Base filter for searching for groups
- ## group_filter="objectclass=groupOfNames"
-
- # The username attribute in the LDAP schema
- ## group_name_attr=cn
+<% if @group_member_attr -%>
+ group_member_attr=<%= @group_member_attr %>
+<% end -%>
+<% end -%>
# Configuration options for specifying the Desktop Database. For more info,
# see http://docs.djangoproject.com/en/1.4/ref/settings/#database-engine
@@ -361,8 +334,7 @@
# Default email address to use for various automated notification from Hue
## default_from_email=hue@localhost
-
-<% if @kerberos_realm != "" %>
+<% if @kerberos_realm != "" -%>
# Configuration options for Kerberos integration for secured Hadoop clusters
# ------------------------------------------------------------------------
[[kerberos]]
@@ -370,12 +342,11 @@
# Path to Hue's Kerberos keytab file
hue_keytab=/etc/hue.keytab
# Kerberos principal name for Hue
- hue_principal=hue/<%= @fqdn %>
+ hue_principal=hue/<%= @fqdn %>@<%= @kerberos_realm %>
# Path to kinit
- kinit_path=<%= (@operatingsystem == 'ubuntu') ? '/usr/bin' : '/usr/kerberos/bin' %>/kinit
-<% end %>
-
+ kinit_path=<%= (@operatingsystem == 'ubuntu' || @operatingsystem == 'Debian') ? '/usr/bin' : '/usr/kerberos/bin' %>/kinit
+<% end -%>
# Configuration options for using OAuthBackend (core) login
# ------------------------------------------------------------------------
[[oauth]]
@@ -616,25 +587,30 @@
# Configuration for YARN (MR2)
# ------------------------------------------------------------------------
[[yarn_clusters]]
-
- [[[default]]]
+<% resourcemanager_hosts = Array(@rm_host) -%>
+<% resourcemanager_hosts.each do |host| -%>
+ [[[<%= host %>]]]
# Enter the host on which you are running the ResourceManager
- resourcemanager_host=<%= @rm_host %>
+ resourcemanager_host=<%= host %>
# The port where the ResourceManager IPC listens on
- resourcemanager_port=<%= @rm_port %>
+ # resourcemanager_port=<% rm_port %>
# Whether to submit jobs to this cluster
submit_to=True
-
+<% if @rm_logical_name -%>
# Resource Manager logical name (required for HA)
- ## logical_name=
+ logical_name=<%= @rm_logical_name %>
+<% else -%>
+ # Resource Manager logical name (required for HA)
+ # logical_name=
+<% end -%>
# Change this if your YARN cluster is Kerberos-secured
security_enabled=<%= if (@kerberos_realm != "") ; "true" else "false" end %>
# URL of the ResourceManager API
- resourcemanager_api_url=<%= @rm_url %>
+ resourcemanager_api_url=http://<%= host %>:<%= @rm_api_port %>
# URL of the ProxyServer API
proxy_api_url=<%= @rm_proxy_url %>
@@ -642,43 +618,12 @@
# URL of the HistoryServer API
history_server_api_url=<%= @history_server_url %>
- # HA support by specifying multiple clusters
- # e.g.
-
- # [[[ha]]]
- # Resource Manager logical name (required for HA)
- # logical_name=my-rm-name
+ # URL of the NodeManager API
+ node_manager_api_url=http://localhost:8042
+<% end -%>
# Configuration for MapReduce (MR1)
# ------------------------------------------------------------------------
- [[mapred_clusters]]
-
- [[[default]]]
- # Enter the host on which you are running the Hadoop JobTracker
- ## jobtracker_host=localhost
-
- # The port where the JobTracker IPC listens on
- ## jobtracker_port=8021
-
- # JobTracker logical name for HA
- ## logical_name=
-
- # Thrift plug-in port for the JobTracker
- ## thrift_port=9290
-
- # Whether to submit jobs to this cluster
- submit_to=False
-
- # Change this if your MapReduce cluster is Kerberos-secured
- ## security_enabled=false
-
- # HA support by specifying multiple clusters
- # e.g.
-
- # [[[ha]]]
- # Enter the logical name of the JobTrackers
- # logical_name=my-jt-name
-
###########################################################################
# Settings to configure the Filebrowser app
@@ -694,10 +639,9 @@
[liboozie]
# The URL where the Oozie service runs on. This is required in order for
- # users to submit jobs. Empty value disables the config check.
+ # users to submit jobs.
oozie_url=<%= @oozie_url %>
- # Requires FQDN in oozie_url if enabled
security_enabled=<%= if (@kerberos_realm != "") ; "true" else "false" end %>
# Location on HDFS where the workflows/coordinator are deployed when submitted.
@@ -716,10 +660,10 @@
## sample_data_dir=...thirdparty/sample_data
# Location on HDFS where the oozie examples and workflows are stored.
- remote_data_dir=/user/hue/oozie/workspaces
+ ## remote_data_dir=/user/hue/oozie/workspaces
# Maximum of Oozie workflows or coodinators to retrieve in one API call.
- oozie_jobs_count=100
+ ## oozie_jobs_count=100
# Use Cron format for defining the frequency of a Coordinator instead of the old frequency number/unit.
## enable_cron_scheduling=true
@@ -731,19 +675,24 @@
[beeswax]
+<% if @hive_host -%>
# Host where HiveServer2 is running.
# If Kerberos security is enabled, use fully-qualified domain name (FQDN).
- ## hive_server_host=localhost
+ hive_server_host=<%= @hive_host %>
# Port where HiveServer2 Thrift server runs on.
- ## hive_server_port=10000
+ hive_server_port=<%= @hive_port %>
# Hive configuration directory, where hive-site.xml is located
- ## hive_conf_dir=/etc/hive/conf
+ hive_conf_dir=/etc/hive/conf
# Timeout in seconds for thrift calls to Hive service
## server_conn_timeout=120
+ # Choose whether Hue uses the GetLog() thrift call to retrieve Hive logs.
+ # If false, Hue will use the FetchResults() thrift call instead.
+ use_get_log_api=false
+
# Set a LIMIT clause when browsing a partitioned table.
# A positive value will be set as the LIMIT. If 0 or negative, do not set any limit.
## browse_partitioned_table_limit=250
@@ -775,7 +724,7 @@
# Choose whether Hue should validate certificates received from the server.
## validate=true
-
+<% end -%>
###########################################################################
# Settings to configure Pig
@@ -797,8 +746,9 @@
# For autocompletion, fill out the librdbms section.
# Sqoop server URL
+<% if @sqoop_url != "" -%>
server_url=<%= @sqoop_url %>
-
+<% end -%>
###########################################################################
# Settings to configure Proxy
@@ -856,8 +806,11 @@
[hbase]
# Comma-separated list of HBase Thrift servers for clusters in the format of '(name|host:port)'.
# Use full hostname with security.
+<% if @hbase_thrift_url != "" -%>
hbase_clusters=(Bigtop|<%= @hbase_thrift_url %>)
-
+<% else -%>
+ ## hbase_clusters=(Cluster|localhost:9090)
+<% end -%>
# HBase configuration directory, where hbase-site.xml is located.
## hbase_conf_dir=/etc/hbase/conf
@@ -877,13 +830,14 @@
[search]
# URL of the Solr Server
+<% if @solr_url != "" -%>
solr_url=<%= @solr_url %>
-
+<% end -%>
# Requires FQDN in solr_url if enabled
## security_enabled=false
## Query sent when no term is entered
- empty_query=*:*
+ ## empty_query=*:*
###########################################################################
@@ -939,10 +893,10 @@
[[[default]]]
# Zookeeper ensemble. Comma separated list of Host/Port.
# e.g. localhost:2181,localhost:2182,localhost:2183
- ## host_ports=localhost:2181
+ host_ports=localhost:2181
# The URL of the REST contrib service (required for znode browsing)
- ## rest_url=http://localhost:9998
+ rest_url=http://localhost:9998
###########################################################################