You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2015/01/13 21:25:17 UTC
ambari git commit: AMBARI-9096. Cluster create fails when
"hive.server2.transport.mode" property is set to "http" (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/trunk c22e1ec08 -> a84acc27f
AMBARI-9096. Cluster create fails when "hive.server2.transport.mode" property is set to "http" (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a84acc27
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a84acc27
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a84acc27
Branch: refs/heads/trunk
Commit: a84acc27f703e3e74e33e8022c0af6df6838b69a
Parents: c22e1ec
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Tue Jan 13 22:24:26 2015 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Tue Jan 13 22:24:26 2015 +0200
----------------------------------------------------------------------
.../libraries/functions/hive_check.py | 5 ++--
.../package/alerts/alert_hive_thrift_port.py | 28 +++++++++++++-------
.../0.12.0.2.0/package/scripts/hive_service.py | 3 ++-
.../HIVE/0.12.0.2.0/package/scripts/params.py | 6 ++++-
.../0.12.0.2.0/package/scripts/service_check.py | 3 ++-
.../python/stacks/2.0.6/configs/default.json | 1 +
.../stacks/2.0.6/configs/default_client.json | 3 ++-
.../python/stacks/2.0.6/configs/secured.json | 3 ++-
.../stacks/2.0.6/configs/secured_client.json | 3 ++-
.../test/python/stacks/2.1/configs/default.json | 3 ++-
.../test/python/stacks/2.1/configs/secured.json | 3 ++-
11 files changed, 42 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py
index 1924c05..f0036be 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py
@@ -22,7 +22,8 @@ from resource_management.core.resources import Execute
from resource_management.libraries.functions import format
import socket
-def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None, smokeuser = 'ambari-qa'):
+def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None, smokeuser = 'ambari-qa',
+ transport_mode = "binary"):
"""
Hive thrift SASL port check
"""
@@ -36,7 +37,7 @@ def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kini
else:
url = format("jdbc:hive2://{address}:{port}")
- if hive_auth != "NOSASL":
+ if hive_auth != "NOSASL" and transport_mode != "http":
cmd = format("! beeline -u '{url}' -e '' ") + "2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'"
Execute(cmd,
user=smokeuser,
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py
index ebfbf55..9979530 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py
@@ -31,6 +31,8 @@ OK_MESSAGE = "TCP OK - %.4f response on port %s"
CRITICAL_MESSAGE = "Connection failed on host {0}:{1}"
HIVE_SERVER_THRIFT_PORT_KEY = '{{hive-site/hive.server2.thrift.port}}'
+HIVE_SERVER_THRIFT_HTTP_PORT_KEY = '{{hive-site/hive.server2.thrift.http.port}}'
+HIVE_SERVER_TRANSPORT_MODE_KEY = '{{hive-site/hive.server2.transport.mode}}'
SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
HIVE_SERVER2_AUTHENTICATION_KEY = '{{hive-site/hive.server2.authentication}}'
HIVE_SERVER_PRINCIPAL_KEY = '{{hive-site/hive.server2.authentication.kerberos.principal}}'
@@ -41,6 +43,7 @@ PERCENT_WARNING = 200
PERCENT_CRITICAL = 200
THRIFT_PORT_DEFAULT = 10000
+HIVE_SERVER_TRANSPORT_MODE_DEFAULT = 'binary'
HIVE_SERVER_PRINCIPAL_DEFAULT = 'hive/_HOST@EXAMPLE.COM'
HIVE_SERVER2_AUTHENTICATION_DEFAULT = 'NOSASL'
SMOKEUSER_KEYTAB_DEFAULT = '/etc/security/keytabs/smokeuser.headless.keytab'
@@ -51,7 +54,8 @@ def get_tokens():
Returns a tuple of tokens in the format {{site/property}} that will be used
to build the dictionary passed into execute
"""
- return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY,HIVE_SERVER2_AUTHENTICATION_KEY,HIVE_SERVER_PRINCIPAL_KEY,SMOKEUSER_KEYTAB_KEY,SMOKEUSER_KEY)
+ return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY,HIVE_SERVER2_AUTHENTICATION_KEY,HIVE_SERVER_PRINCIPAL_KEY,
+ SMOKEUSER_KEYTAB_KEY,SMOKEUSER_KEY,HIVE_SERVER_THRIFT_HTTP_PORT_KEY,HIVE_SERVER_TRANSPORT_MODE_KEY)
def execute(parameters=None, host_name=None):
@@ -66,9 +70,15 @@ def execute(parameters=None, host_name=None):
if parameters is None:
return (('UNKNOWN', ['There were no parameters supplied to the script.']))
- thrift_port = THRIFT_PORT_DEFAULT
- if HIVE_SERVER_THRIFT_PORT_KEY in parameters:
- thrift_port = int(parameters[HIVE_SERVER_THRIFT_PORT_KEY])
+ transport_mode = HIVE_SERVER_TRANSPORT_MODE_DEFAULT
+ if HIVE_SERVER_TRANSPORT_MODE_KEY in parameters:
+ transport_mode = parameters[HIVE_SERVER_TRANSPORT_MODE_KEY]
+
+ port = THRIFT_PORT_DEFAULT
+ if transport_mode.lower() == 'binary' and HIVE_SERVER_THRIFT_PORT_KEY in parameters:
+ port = int(parameters[HIVE_SERVER_THRIFT_PORT_KEY])
+ elif transport_mode.lower() == 'http' and HIVE_SERVER_THRIFT_HTTP_PORT_KEY in parameters:
+ port = int(parameters[HIVE_SERVER_THRIFT_HTTP_PORT_KEY])
security_enabled = False
if SECURITY_ENABLED_KEY in parameters:
@@ -103,8 +113,8 @@ def execute(parameters=None, host_name=None):
start_time = time.time()
try:
- hive_check.check_thrift_port_sasl(host_name, thrift_port, hive_server2_authentication,
- hive_server_principal, kinitcmd, smokeuser)
+ hive_check.check_thrift_port_sasl(host_name, port, hive_server2_authentication,
+ hive_server_principal, kinitcmd, smokeuser, transport_mode = transport_mode)
is_thrift_port_ok = True
except:
is_thrift_port_ok = False
@@ -112,13 +122,13 @@ def execute(parameters=None, host_name=None):
if is_thrift_port_ok == True:
result_code = 'OK'
total_time = time.time() - start_time
- label = OK_MESSAGE % (total_time, thrift_port)
+ label = OK_MESSAGE % (total_time, port)
else:
result_code = 'CRITICAL'
- label = CRITICAL_MESSAGE.format(host_name,thrift_port)
+ label = CRITICAL_MESSAGE.format(host_name,port)
except Exception, e:
label = str(e)
result_code = 'UNKNOWN'
- return ((result_code, [label]))
\ No newline at end of file
+ return ((result_code, [label]))
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
index 1120b5d..3b32977 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
@@ -82,7 +82,8 @@ def hive_service(name, action='start', rolling_restart=False):
while time.time() < end_time:
try:
check_thrift_port_sasl(address, port, params.hive_server2_authentication,
- params.hive_server_principal, kinitcmd, params.smokeuser)
+ params.hive_server_principal, kinitcmd, params.smokeuser,
+ transport_mode=params.hive_transport_mode)
is_service_socket_valid = True
break
except Exception, e:
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
index a027d52..00bada8 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py
@@ -135,7 +135,11 @@ hive_var_lib = '/var/lib/hive'
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
hive_server_host = config['clusterHostInfo']['hive_server_host'][0]
hive_server_hosts = config['clusterHostInfo']['hive_server_host']
-hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
+hive_transport_mode = config['configurations']['hive-site']['hive.server2.transport.mode']
+if hive_transport_mode.lower() == "http":
+ hive_server_port = config['configurations']['hive-site']['hive.server2.thrift.http.port']
+else:
+ hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}")
hive_server_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal']
hive_server2_authentication = config['configurations']['hive-site']['hive.server2.authentication']
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py
index 4e8b886..09d31aa 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py
@@ -42,7 +42,8 @@ class HiveServiceCheck(Script):
for address in address_list:
try:
check_thrift_port_sasl(address, port, params.hive_server2_authentication,
- params.hive_server_principal, kinitcmd, params.smokeuser)
+ params.hive_server_principal, kinitcmd, params.smokeuser,
+ transport_mode=params.hive_transport_mode)
print "Successfully connected to %s on port %s" % (address, port)
workable_server_available = True
except:
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 82b485b..9132b2f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -370,6 +370,7 @@
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
"hive.server2.enable.doAs": "true",
"hive.server2.authentication": "NOSASL",
+ "hive.server2.transport.mode": "binary",
"hive.optimize.mapjoin.mapreduce": "true"
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json
index f544b88..8e643de 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json
@@ -364,7 +364,8 @@
"ambari.hive.db.schema.name": "hive",
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
- "hive.server2.enable.doAs": "true",
+ "hive.server2.enable.doAs": "true",
+ "hive.server2.transport.mode": "binary",
"hive.optimize.mapjoin.mapreduce": "true"
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
index d48b0ab..119ab88 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
@@ -407,7 +407,8 @@
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
"hive.server2.enable.doAs": "true",
- "hive.optimize.mapjoin.mapreduce": "true",
+ "hive.optimize.mapjoin.mapreduce": "true",
+ "hive.server2.transport.mode": "binary",
"hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab"
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json
index 69c7b4d..4faf0f8 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json
@@ -405,7 +405,8 @@
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
"hive.server2.enable.doAs": "true",
- "hive.optimize.mapjoin.mapreduce": "true",
+ "hive.optimize.mapjoin.mapreduce": "true",
+ "hive.server2.transport.mode": "binary",
"hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab"
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.1/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/default.json b/ambari-server/src/test/python/stacks/2.1/configs/default.json
index ac829dd..689279d 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/default.json
@@ -479,7 +479,8 @@
"ambari.hive.db.schema.name": "hive",
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
- "hive.server2.enable.doAs": "true",
+ "hive.server2.enable.doAs": "true",
+ "hive.server2.transport.mode": "binary",
"hive.optimize.mapjoin.mapreduce": "true"
},
"yarn-site": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.1/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured.json b/ambari-server/src/test/python/stacks/2.1/configs/secured.json
index 0ba831d..b2a6053 100644
--- a/ambari-server/src/test/python/stacks/2.1/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.1/configs/secured.json
@@ -451,7 +451,8 @@
"hive.metastore.execute.setugi": "true",
"hive.auto.convert.sortmerge.join.noconditionaltask": "true",
"hive.server2.enable.doAs": "true",
- "hive.optimize.mapjoin.mapreduce": "true",
+ "hive.optimize.mapjoin.mapreduce": "true",
+ "hive.server2.transport.mode": "binary",
"hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab"
},
"yarn-site": {