You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/11/07 22:41:25 UTC
ambari git commit: AMBARI-8213. ambari cluster HS2 health check
causing errors in logs (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-1.7.0 1bb547592 -> 0798ea5a7
AMBARI-8213. ambari cluster HS2 health check causing errors in logs (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0798ea5a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0798ea5a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0798ea5a
Branch: refs/heads/branch-1.7.0
Commit: 0798ea5a7761b5a027927313d3e7a5fdd491bae5
Parents: 1bb5475
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Nov 7 23:41:16 2014 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Nov 7 23:41:16 2014 +0200
----------------------------------------------------------------------
.../libraries/functions/hive_check.py | 90 ++++++--------------
.../HIVE/package/scripts/hive_service.py | 11 ++-
.../services/HIVE/package/scripts/params.py | 2 +
.../HIVE/package/scripts/service_check.py | 12 ++-
.../package/files/check_hive_thrift_port.py | 17 ++--
.../services/NAGIOS/package/scripts/params.py | 10 +++
.../package/templates/hadoop-commands.cfg.j2 | 2 +-
.../package/templates/hadoop-services.cfg.j2 | 2 +-
.../HIVE/package/scripts/hive_service.py | 11 ++-
.../services/HIVE/package/scripts/params.py | 2 +
.../HIVE/package/scripts/service_check.py | 12 ++-
.../package/files/check_hive_thrift_port.py | 17 ++--
.../services/NAGIOS/package/scripts/params.py | 10 +++
.../package/templates/hadoop-commands.cfg.j2 | 2 +-
.../package/templates/hadoop-services.cfg.j2 | 2 +-
.../stacks/1.3.2/HIVE/test_hive_server.py | 16 ++--
.../1.3.2/HIVE/test_hive_service_check.py | 13 ++-
.../python/stacks/1.3.2/configs/default.json | 3 +-
.../python/stacks/1.3.2/configs/secured.json | 5 +-
.../stacks/2.0.6/HIVE/test_hive_server.py | 23 ++---
.../2.0.6/HIVE/test_hive_service_check.py | 9 +-
.../python/stacks/2.0.6/configs/default.json | 3 +-
.../python/stacks/2.0.6/configs/secured.json | 3 +-
23 files changed, 152 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py
index 282fed2..77a36e6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py
@@ -17,72 +17,36 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-import socket
from resource_management.core.exceptions import Fail
+from resource_management.core.resources import Execute
+from resource_management.libraries.functions import format
+import socket
-def check_thrift_port_sasl(address, port, timeout = 5, security_enabled = False):
+def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None):
"""
Hive thrift SASL port check
"""
+ BEELINE_CHECK_TIMEOUT = 7
+
+ if kinitcmd:
+ url = format("jdbc:hive2://{address}:{port}/;principal={key}")
+ Execute(kinitcmd)
+ else:
+ url = format("jdbc:hive2://{address}:{port}")
+
+ if hive_auth != "NOSASL":
+ cmd = format("! beeline -u '{url}' -e '' ") + "2>&1| awk '{print}'|grep Error"
+ Execute(cmd,
+ path=["/bin/", "/usr/bin/", "/usr/lib/hive/bin/", "/usr/sbin/"],
+ timeout=BEELINE_CHECK_TIMEOUT
+ )
+ else:
+ s = socket.socket()
+ s.settimeout(1)
+ try:
+ s.connect((address, port))
+ except socket.error, e:
+ raise
+ finally:
+ s.close()
- #Authentification mechanism
- mechanism = "PLAIN"
- #Anonymous username
- usr = "ANONYMOUS"
- start_byte = 0x01 #START communication
- ok_byte = 0x02 #OK
- bad_byte = 0x03 #BAD
- error_byte = 0x04 #ERROR
- complete_byte = 0x05 #COMPLETE communication
-
- msg = bytearray()
-
- msg.append(start_byte)
- msg.append(0)
- msg.append(0)
- msg.append(0)
- msg.append(len(mechanism))
- for elem in mechanism:
- msg.append(ord(elem))
-
- msg.append(ok_byte)
- msg.append(0)
- msg.append(0)
- msg.append(0)
- msg.append(len(usr)*2+2)
-
- #Adding anonymous user name
- msg.append(0)
- for elem in usr:
- msg.append(ord(elem))
-
- #Adding anonymous user password
- msg.append(0)
- for elem in usr:
- msg.append(ord(elem))
-
- msg.append(complete_byte)
- msg.append(0)
- msg.append(0)
- msg.append(0)
- msg.append(0)
-
- is_service_socket_valid = False
- s = socket.socket()
- s.settimeout(timeout)
-
- try:
- s.connect((address, port))
- #Successfull connection, port check passed
- is_service_socket_valid = True
-
- # Try to send anonymous plain auth message to thrift to prevent errors in hive log
- # Plain mechanism is not supported in security mode
- if not security_enabled:
- s.send(msg)
- except socket.error, e:
- #Expected if service unreachable
- pass
- finally:
- s.close()
- return is_service_socket_valid
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py
index d3bd71b..33bfd79 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py
@@ -68,11 +68,18 @@ def hive_service(
is_service_socket_valid = False
print "Waiting for the Hive server to start..."
+ if params.security_enabled:
+ kinitcmd=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
+ else:
+ kinitcmd=None
+
while time.time() < end_time:
- if check_thrift_port_sasl(address, port, 2, security_enabled=params.security_enabled):
+ try:
+ check_thrift_port_sasl(address, port, params.hive_server2_authentication,
+ params.hive_server_principal, kinitcmd)
is_service_socket_valid = True
break
- else:
+ except:
time.sleep(2)
elapsed_time = time.time() - start_time
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py
index f27c28c..69babb0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py
@@ -56,6 +56,8 @@ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
hive_server_host = config['clusterHostInfo']['hive_server_host'][0]
hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}")
+hive_server_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal']
+hive_server2_authentication = config['configurations']['hive-site']['hive.server2.authentication']
smokeuser = config['configurations']['cluster-env']['smokeuser']
smoke_test_sql = format("{tmp_dir}/hiveserver2.sql")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py
index b75578b..bd7537b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py
@@ -33,9 +33,17 @@ class HiveServiceCheck(Script):
address=format("{hive_server_host}")
port=int(format("{hive_server_port}"))
print "Test connectivity to hive server"
- if check_thrift_port_sasl(address, port, security_enabled=params.security_enabled):
- print "Successfully connected to %s on port %s" % (address, port)
+
+ if params.security_enabled:
+ kinitcmd=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
else:
+ kinitcmd=None
+
+ try:
+ check_thrift_port_sasl(address, port, params.hive_server2_authentication,
+ params.hive_server_principal, kinitcmd)
+ print "Successfully connected to %s on port %s" % (address, port)
+ except:
print "Connection to %s on port %s failed" % (address, port)
exit(1)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py
index c9414f7..c25339d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py
@@ -37,7 +37,9 @@ def main():
parser.add_option("-H", "--host", dest="address", help="Hive thrift host")
parser.add_option("-p", "--port", type="int", dest="port", help="Hive thrift port")
- parser.add_option("--security-enabled", action="store_true", dest="security_enabled")
+ parser.add_option("-a", "--hive-auth", dest="hive_auth")
+ parser.add_option("-k", "--hive-principal", dest="key")
+ parser.add_option("-c", "--cmd-kinit", dest="kinitcmd")
(options, args) = parser.parse_args()
@@ -49,23 +51,20 @@ def main():
print "Specify hive thrift port (--port or -p)"
exit(-1)
- if options.security_enabled:
- security_enabled = options.security_enabled
- else:
- security_enabled = False
-
address = options.address
port = options.port
starttime = time()
- if check_thrift_port_sasl(address, port, security_enabled=security_enabled):
+ try:
+ with Environment() as env:
+ check_thrift_port_sasl(address, port, options.hive_auth, options.key, options.kinitcmd)
timetaken = time() - starttime
print OK_MESSAGE % (timetaken, port)
- exit(0)
- else:
+ except:
print CRITICAL_MESSAGE % (address, port)
exit(2)
+ exit(0)
if __name__ == "__main__":
main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py
index 8eeb181..6b4dd67 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/params.py
@@ -84,6 +84,16 @@ security_enabled = config['configurations']['cluster-env']['security_enabled']
nagios_keytab_path = default("/configurations/nagios-env/nagios_keytab_path", "/etc/security/keytabs/nagios.service.keytab")
kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+if security_enabled:
+ smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+ smokeuser = config['configurations']['cluster-env']['smokeuser']
+ kinitcmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
+ hive_server_principal = default("configurations/hive-site/hive.server2.authentication.kerberos.principal","hive/_HOST@EXAMPLE.COM")
+else:
+ kinitcmd = ''
+ hive_server_principal = ''
+hive_server2_authentication = default("configurations/hive-site/hive.server2.authentication","NOSASL")
+
ganglia_port = "8651"
ganglia_collector_slaves_port = "8660"
ganglia_collector_namenode_port = "8661"
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2
index ff13c0d..c2d6434 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2
@@ -143,5 +143,5 @@ define command{
define command{
command_name check_tcp_wrapper_sasl
- command_line /var/lib/ambari-agent/ambari-python-wrap $USER1$/mm_wrapper.py legacy_check_wrapper -- /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$ $ARG2$
+ command_line /var/lib/ambari-agent/ambari-python-wrap $USER1$/mm_wrapper.py and $HOSTADDRESS$ -- /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H ^^ -p $ARG1$ -a $ARG2$ -k $ARG3$ -c '$ARG4$'
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2
index 48bba18..0e4e2dc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2
@@ -553,7 +553,7 @@ define service {
use hadoop-service
service_description HIVE-SERVER::HiveServer2 process
servicegroups HIVE
- check_command check_tcp_wrapper_sasl!{{ hive_server_port }}!{{ '--security-enabled' if security_enabled else '' }}!-w 1 -c 1
+ check_command check_tcp_wrapper_sasl!{{ hive_server_port }}!{{ hive_server2_authentication }}!{{ hive_server_principal }}!{{ kinitcmd }}
_host_component HIVE_SERVER
normal_check_interval 0.5
retry_check_interval 0.5
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py
index 294121d..54de40a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py
@@ -74,11 +74,18 @@ def hive_service(
is_service_socket_valid = False
print "Waiting for the Hive server to start..."
+ if params.security_enabled:
+ kinitcmd=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
+ else:
+ kinitcmd=None
+
while time.time() < end_time:
- if check_thrift_port_sasl(address, port, 2, security_enabled=params.security_enabled):
+ try:
+ check_thrift_port_sasl(address, port, params.hive_server2_authentication,
+ params.hive_server_principal, kinitcmd)
is_service_socket_valid = True
break
- else:
+ except:
time.sleep(2)
elapsed_time = time.time() - start_time
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
index c1279d8..5879cea 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
@@ -105,6 +105,8 @@ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
hive_server_host = config['clusterHostInfo']['hive_server_host'][0]
hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}")
+hive_server_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal']
+hive_server2_authentication = config['configurations']['hive-site']['hive.server2.authentication']
smokeuser = config['configurations']['cluster-env']['smokeuser']
smoke_test_sql = format("{tmp_dir}/hiveserver2.sql")
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py
index b75578b..bd7537b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py
@@ -33,9 +33,17 @@ class HiveServiceCheck(Script):
address=format("{hive_server_host}")
port=int(format("{hive_server_port}"))
print "Test connectivity to hive server"
- if check_thrift_port_sasl(address, port, security_enabled=params.security_enabled):
- print "Successfully connected to %s on port %s" % (address, port)
+
+ if params.security_enabled:
+ kinitcmd=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
else:
+ kinitcmd=None
+
+ try:
+ check_thrift_port_sasl(address, port, params.hive_server2_authentication,
+ params.hive_server_principal, kinitcmd)
+ print "Successfully connected to %s on port %s" % (address, port)
+ except:
print "Connection to %s on port %s failed" % (address, port)
exit(1)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py
index c9414f7..c25339d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py
@@ -37,7 +37,9 @@ def main():
parser.add_option("-H", "--host", dest="address", help="Hive thrift host")
parser.add_option("-p", "--port", type="int", dest="port", help="Hive thrift port")
- parser.add_option("--security-enabled", action="store_true", dest="security_enabled")
+ parser.add_option("-a", "--hive-auth", dest="hive_auth")
+ parser.add_option("-k", "--hive-principal", dest="key")
+ parser.add_option("-c", "--cmd-kinit", dest="kinitcmd")
(options, args) = parser.parse_args()
@@ -49,23 +51,20 @@ def main():
print "Specify hive thrift port (--port or -p)"
exit(-1)
- if options.security_enabled:
- security_enabled = options.security_enabled
- else:
- security_enabled = False
-
address = options.address
port = options.port
starttime = time()
- if check_thrift_port_sasl(address, port, security_enabled=security_enabled):
+ try:
+ with Environment() as env:
+ check_thrift_port_sasl(address, port, options.hive_auth, options.key, options.kinitcmd)
timetaken = time() - starttime
print OK_MESSAGE % (timetaken, port)
- exit(0)
- else:
+ except:
print CRITICAL_MESSAGE % (address, port)
exit(2)
+ exit(0)
if __name__ == "__main__":
main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py
index 778d830..a8bc82f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/params.py
@@ -228,6 +228,16 @@ security_enabled = config['configurations']['cluster-env']['security_enabled']
nagios_keytab_path = default("/configurations/nagios-env/nagios_keytab_path", "/etc/security/keytabs/nagios.service.keytab")
kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+if security_enabled:
+ smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+ smokeuser = config['configurations']['cluster-env']['smokeuser']
+ kinitcmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
+ hive_server_principal = default("configurations/hive-site/hive.server2.authentication.kerberos.principal","hive/_HOST@EXAMPLE.COM")
+else:
+ kinitcmd = ''
+ hive_server_principal = ''
+hive_server2_authentication = default("configurations/hive-site/hive.server2.authentication","NOSASL")
+
dfs_ha_enabled = False
dfs_ha_nameservices = default("/configurations/hdfs-site/dfs.nameservices", None)
dfs_ha_namenode_ids = default(format("/configurations/hdfs-site/dfs.ha.namenodes.{dfs_ha_nameservices}"), None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2
index c1a792c..64840b1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2
@@ -157,7 +157,7 @@ define command{
define command{
command_name check_tcp_wrapper_sasl
- command_line /var/lib/ambari-agent/ambari-python-wrap $USER1$/mm_wrapper.py and $HOSTADDRESS$ -- /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H ^^ -p $ARG1$ $ARG2$
+ command_line /var/lib/ambari-agent/ambari-python-wrap $USER1$/mm_wrapper.py and $HOSTADDRESS$ -- /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H ^^ -p $ARG1$ -a $ARG2$ -k $ARG3$ -c '$ARG4$'
}
define command{
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2
index 0d8b2e8..24c66cd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2
@@ -726,7 +726,7 @@ define service {
use hadoop-service
service_description HIVE-SERVER::HiveServer2 process
servicegroups HIVE
- check_command check_tcp_wrapper_sasl!{{ hive_server_port }}!{{ '--security-enabled' if security_enabled else '' }}!-w 1 -c 1
+ check_command check_tcp_wrapper_sasl!{{ hive_server_port }}!{{ hive_server2_authentication }}!{{ hive_server_principal }}!{{ kinitcmd }}
_host_component HIVE_SERVER
normal_check_interval 0.5
retry_check_interval 0.5
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py
index 4dceec1..e49bdba 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py
@@ -80,10 +80,12 @@ class TestHiveServer(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10
)
+ self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000' -e '' 2>&1| awk '{print}'|grep Error",
+ path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+ timeout = 7,
+ )
- self.assertNoMoreResources()
- self.assertTrue(socket_mock.called)
- self.assertTrue(s.close.called)
+ self.assertNoMoreResources()
@patch("socket.socket")
def test_stop_default(self, socket_mock):
@@ -156,10 +158,12 @@ class TestHiveServer(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10
)
-
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; ',)
+ self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep Error",
+ path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+ timeout = 7,
+ )
self.assertNoMoreResources()
- self.assertTrue(socket_mock.called)
- self.assertTrue(s.close.called)
@patch("socket.socket")
def test_stop_secured(self, socket_mock):
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py
index ad75ce1..2934ee9 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py
@@ -33,6 +33,10 @@ class TestServiceCheck(RMFTestCase):
command="service_check",
config_file="default.json"
)
+ self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000' -e '' 2>&1| awk '{print}'|grep Error",
+ path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+ timeout = 7,
+ )
self.assertResourceCalled('File', '/tmp/hcatSmoke.sh',
content = StaticFile('hcatSmoke.sh'),
mode = 0755,
@@ -70,7 +74,6 @@ class TestServiceCheck(RMFTestCase):
try_sleep = 5,
)
self.assertNoMoreResources()
- self.assertTrue(socket_mock.called)
@patch("sys.exit")
def test_service_check_secured(self, sys_exit_mock, socket_mock):
@@ -80,6 +83,11 @@ class TestServiceCheck(RMFTestCase):
command="service_check",
config_file="secured.json"
)
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; ',)
+ self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep Error",
+ path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+ timeout = 7,
+ )
self.assertResourceCalled('File', '/tmp/hcatSmoke.sh',
content = StaticFile('hcatSmoke.sh'),
mode = 0755,
@@ -117,5 +125,4 @@ class TestServiceCheck(RMFTestCase):
tries = 3,
try_sleep = 5,
)
- self.assertNoMoreResources()
- self.assertTrue(socket_mock.called)
+ self.assertNoMoreResources()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
index d9f9f71..ed1c861 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
@@ -281,7 +281,8 @@
"ambari.hive.db.schema.name": "hive",
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
- "hive.server2.enable.doAs": "true",
+ "hive.server2.enable.doAs": "true",
+ "hive.server2.authentication": "NONE",
"hive.optimize.mapjoin.mapreduce": "true"
},
"webhcat-env": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json b/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json
index 3c08702..5bfbea8 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/secured.json
@@ -453,8 +453,9 @@
"ambari.hive.db.schema.name": "hive",
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
- "hive.server2.enable.doAs": "true",
- "hive.optimize.mapjoin.mapreduce": "true",
+ "hive.server2.enable.doAs": "true",
+ "hive.server2.authentication": "NONE",
+ "hive.optimize.mapjoin.mapreduce": "true",
"hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab"
},
"webhcat-env": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index 581b4d3..cf4f062 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -39,7 +39,7 @@ class TestHiveServer(RMFTestCase):
@patch("socket.socket")
def test_start_default(self, socket_mock, popen_mock):
s = socket_mock.return_value
-
+
self.executeScript("2.0.6/services/HIVE/package/scripts/hive_server.py",
classname = "HiveServer",
command = "start",
@@ -58,15 +58,12 @@ class TestHiveServer(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10
)
-
self.assertNoMoreResources()
self.assertTrue(popen_mock.called)
popen_mock.assert_called_with(
['su', '-s', '/bin/bash', '-', u'hive', '-c', "metatool -listFSRoot 2>/dev/null | grep hdfs:// | grep -v '.db$'"],
shell=False, preexec_fn=None, stderr=-2, stdout=-1, env=None, cwd=None
)
- self.assertTrue(socket_mock.called)
- self.assertTrue(s.close.called)
@patch("socket.socket")
def test_stop_default(self, socket_mock):
@@ -81,7 +78,6 @@ class TestHiveServer(RMFTestCase):
)
self.assertNoMoreResources()
- self.assertFalse(socket_mock.called)
def test_configure_secured(self):
@@ -94,10 +90,7 @@ class TestHiveServer(RMFTestCase):
self.assertNoMoreResources()
@patch("hive_service.check_fs_root")
- @patch("socket.socket")
- def test_start_secured(self, socket_mock, check_fs_root_mock):
- s = socket_mock.return_value
-
+ def test_start_secured(self, check_fs_root_mock):
self.executeScript("2.0.6/services/HIVE/package/scripts/hive_server.py",
classname = "HiveServer",
command = "start",
@@ -115,14 +108,15 @@ class TestHiveServer(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10
)
-
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; ',)
+ self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep Error",
+ path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+ timeout=7
+ )
self.assertNoMoreResources()
self.assertTrue(check_fs_root_mock.called)
- self.assertTrue(socket_mock.called)
- self.assertTrue(s.close.called)
- @patch("socket.socket")
- def test_stop_secured(self, socket_mock):
+ def test_stop_secured(self):
self.executeScript("2.0.6/services/HIVE/package/scripts/hive_server.py",
classname = "HiveServer",
command = "stop",
@@ -134,7 +128,6 @@ class TestHiveServer(RMFTestCase):
)
self.assertNoMoreResources()
- self.assertFalse(socket_mock.called)
def assert_configure_default(self):
self.assertResourceCalled('HdfsDirectory', '/apps/tez/',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py
index 6ba4d48..99b0b33 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py
@@ -72,7 +72,6 @@ class TestServiceCheck(RMFTestCase):
try_sleep = 5,
)
self.assertNoMoreResources()
- self.assertTrue(socket_mock.called)
@patch("sys.exit")
def test_service_check_secured(self, sys_exit_mock, socket_mock):
@@ -82,6 +81,11 @@ class TestServiceCheck(RMFTestCase):
command="service_check",
config_file="secured.json"
)
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; ',)
+ self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep Error",
+ path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+ timeout= 7
+ )
self.assertResourceCalled('File', '/tmp/hcatSmoke.sh',
content = StaticFile('hcatSmoke.sh'),
mode = 0755,
@@ -121,5 +125,4 @@ class TestServiceCheck(RMFTestCase):
tries = 3,
try_sleep = 5,
)
- self.assertNoMoreResources()
- self.assertTrue(socket_mock.called)
+ self.assertNoMoreResources()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 83e36ae..4896410 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -368,7 +368,8 @@
"ambari.hive.db.schema.name": "hive",
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
- "hive.server2.enable.doAs": "true",
+ "hive.server2.enable.doAs": "true",
+ "hive.server2.authentication": "NOSASL",
"hive.optimize.mapjoin.mapreduce": "true"
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/0798ea5a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
index dba36e5..4f71f2e 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
@@ -406,7 +406,8 @@
"ambari.hive.db.schema.name": "hive",
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
- "hive.server2.enable.doAs": "true",
+ "hive.server2.enable.doAs": "true",
+ "hive.server2.authentication": "NONE",
"hive.optimize.mapjoin.mapreduce": "true",
"hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab"
},