You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2016/04/20 01:59:05 UTC
ambari git commit: AMBARI-15955. Hive Server Interactive to also
write configs to conf.server and specify folder to unpack LLAP Slider app
(alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk 86dece09b -> a02ae60d1
AMBARI-15955. Hive Server Interactive to also write configs to conf.server and specify folder to unpack LLAP Slider app (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a02ae60d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a02ae60d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a02ae60d
Branch: refs/heads/trunk
Commit: a02ae60d1ac18d270e85318756d6d6b3b6f0dcee
Parents: 86dece0
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Mon Apr 18 15:59:03 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Tue Apr 19 16:58:30 2016 -0700
----------------------------------------------------------------------
.../libraries/functions/copy_tarball.py | 2 -
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 22 +++--
.../package/scripts/hive_interactive.py | 39 +++++----
.../package/scripts/hive_metastore.py | 2 +-
.../package/scripts/hive_server_interactive.py | 17 ++--
.../0.12.0.2.0/package/scripts/hive_service.py | 6 +-
.../package/scripts/hive_service_interactive.py | 2 +-
.../0.12.0.2.0/package/scripts/params_linux.py | 35 +++++---
.../0.12.0.2.0/package/scripts/status_params.py | 8 +-
.../main/resources/scripts/Ambaripreupload.py | 3 +-
.../stacks/2.5/HIVE/test_hive_server_int.py | 86 +++++++++++++++-----
.../python/stacks/2.5/configs/hsi_default.json | 6 +-
12 files changed, 153 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index 38e4fb4..286df8d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -50,8 +50,6 @@ TARBALL_MAP = {
"/{0}/apps/{1}/tez_hive2/tez.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
"hive": ("{0}/{1}/hive/hive.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
"/{0}/apps/{1}/hive/hive.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
- "hive2": ("{0}/{1}/hive2/hive.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
- "/{0}/apps/{1}/hive2/hive.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
"pig": ("{0}/{1}/pig/pig.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
"/{0}/apps/{1}/pig/pig.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
"hadoop_streaming": ("{0}/{1}/hadoop-mapreduce/hadoop-streaming.jar".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index a2ee491..a57e43a 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -215,6 +215,7 @@ def hive(name=None):
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
+ Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
@@ -257,8 +258,8 @@ def hive(name=None):
content=Template("hive.conf.j2")
)
- if (name == 'metastore' or name == 'hiveserver2') and params.target != None and not os.path.exists(params.target):
- jdbc_connector()
+ if (name == 'metastore' or name == 'hiveserver2') and params.target_hive is not None and not os.path.exists(params.target_hive):
+ jdbc_connector(params.target_hive)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
@@ -386,7 +387,11 @@ def fill_conf_dir(component_conf_dir):
)
-def jdbc_connector():
+def jdbc_connector(target):
+ """
+ Shared by Hive Batch, Hive Metastore, and Hive Interactive
+ :param target: Target of jdbc jar name, which could be for any of the components above.
+ """
import params
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
@@ -418,19 +423,20 @@ def jdbc_connector():
Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
else:
- Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
- #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
+ Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target),
+ #creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo = True)
else:
#for default hive db (Mysql)
- Execute(('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), params.target),
- #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
+ Execute(('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), target),
+ #creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True
)
+ pass
- File(params.target,
+ File(target,
mode = 0644,
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
index 9cfd41f..24047ff 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py
@@ -18,13 +18,17 @@ limitations under the License.
"""
+# Python Imports
import os
import glob
from urlparse import urlparse
+# Resource Management and Common Imports
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions.version import compare_versions
from resource_management.core.resources.service import ServiceConfig
from resource_management.core.resources.system import File, Execute, Directory
@@ -63,26 +67,23 @@ def hive_interactive(name=None):
'hive.enforce.sorting']
# Copy Tarballs in HDFS.
- resource_created = copy_to_hdfs("tez_hive2",
- params.user_group,
- params.hdfs_user,
- file_mode=params.tarballs_mode,
- host_sys_prepped=params.host_sys_prepped)
+ if params.stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted_major):
+ resource_created = copy_to_hdfs("tez_hive2",
+ params.user_group,
+ params.hdfs_user,
+ file_mode=params.tarballs_mode,
+ host_sys_prepped=params.host_sys_prepped)
- copy_to_hdfs("hive2",
- params.user_group,
- params.hdfs_user,
- file_mode=params.tarballs_mode,
- host_sys_prepped=params.host_sys_prepped) or resource_created
-
- if resource_created:
- params.HdfsResource(None, action="execute")
+ if resource_created:
+ params.HdfsResource(None, action="execute")
Directory(params.hive_interactive_etc_dir_prefix,
mode=0755
)
- fill_conf_dir(params.hive_server_interactive_conf_dir)
+ Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list))
+ for conf_dir in params.hive_conf_dirs_list:
+ fill_conf_dir(conf_dir)
'''
As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
@@ -105,6 +106,14 @@ def hive_interactive(name=None):
group=params.user_group,
mode=0644)
+ XmlConfig("hive-site.xml",
+ conf_dir=os.path.dirname(params.hive_server_interactive_conf_dir),
+ configurations=merged_hive_interactive_site,
+ configuration_attributes=params.config['configuration_attributes']['hive-interactive-site'],
+ owner=params.hive_user,
+ group=params.user_group,
+ mode=0644)
+
# Merge tez-interactive with tez-site
XmlConfig("tez-site.xml",
conf_dir = params.tez_interactive_config_dir,
@@ -135,7 +144,7 @@ def hive_interactive(name=None):
)
if not os.path.exists(params.target_hive_interactive):
- jdbc_connector()
+ jdbc_connector(params.target_hive_interactive)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index 8adff0a..319653e 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -200,7 +200,7 @@ class HiveMetastoreDefault(HiveMetastore):
# download it if it does not exist
if not os.path.exists(params.source_jdbc_file):
- jdbc_connector()
+ jdbc_connector(params.target_hive)
target_directory_and_filename = os.path.join(target_directory, os.path.basename(params.source_jdbc_file))
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 9d011a0..0a2105d 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -23,6 +23,7 @@ import os
import re
import time
import shutil
+from datetime import datetime
# Ambari Commons & Resource Management imports
from resource_management.libraries.script.script import Script
@@ -197,16 +198,17 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
# TODO, start only if not already running.
# TODO : Currently hardcoded the params. Need to read the suggested values from hive2/hive-site.xml.
- # TODO, ensure that script works as hive from cmd when not cd'ed in /homve/hive
+ # TODO, ensure that script works as hive from cmd when not cd'ed in /home/hive
# Needs permission to write to hive home dir.
- cmd = ''
+ unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
+
+ cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 "
+ "-slider-am-container-mb {slider_am_container_mb} --loglevel INFO --output {unique_name}")
+
if params.security_enabled:
- cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb "
- "{slider_am_container_mb} --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
- "{hive_llap_keytab_file} --slider-principal {hive_headless_keytab} --loglevel INFO")
- else:
- cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances 1 -slider-am-container-mb {slider_am_container_mb} --loglevel INFO")
+ cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
+ "{hive_llap_keytab_file} --slider-principal {hive_headless_keytab}")
run_file_path = None
try:
@@ -234,6 +236,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
# TODO : Sleep below is not a good idea. We need to check the status of LLAP app to figure out it got
# launched properly and is in running state. Then go ahead with Hive Interactive Server start.
Logger.info("Sleeping for 30 secs")
+ # Important to mock this sleep call during unit tests.
time.sleep(30)
Logger.info("LLAP app deployed successfully.")
return True
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
index 33e8e47..8399f9c 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py
@@ -101,13 +101,13 @@ def hive_service(name, action='start', upgrade_type=None):
params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
db_connection_check_command = format(
- "{java64_home}/bin/java -cp {check_db_connection_jar}:{target} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
+ "{java64_home}/bin/java -cp {check_db_connection_jar}:{target_hive} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
try:
Execute(db_connection_check_command,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10)
except:
- show_logs(params.hive_log_dir, user)
+ show_logs(params.hive_log_dir, params.hive_user)
raise
elif action == 'stop':
@@ -131,7 +131,7 @@ def hive_service(name, action='start', upgrade_type=None):
try_sleep=3,
)
except:
- show_logs(params.hive_log_dir, user)
+ show_logs(params.hive_log_dir, params.hive_user)
raise
File(pid_file,
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py
index 92e67c7..daf0301 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py
@@ -67,7 +67,7 @@ def hive_service_interactive(name, action='start', upgrade_type=None):
params.hive_jdbc_driver == "org.postgresql.Driver" or \
params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
db_connection_check_command = format(
- "{java64_home}/bin/java -cp {check_db_connection_jar}:{target} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
+ "{java64_home}/bin/java -cp {check_db_connection_jar}:{target_hive_interactive} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
Execute(db_connection_check_command,
path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10)
elif action == 'stop':
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index 9bee786..ca227b5 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -199,7 +199,7 @@ hive_use_existing_db = hive_database.startswith('Existing')
# NOT SURE THAT IT'S A GOOD IDEA TO USE PATH TO CLASS IN DRIVER, MAYBE IT WILL BE BETTER TO USE DB TYPE.
# BECAUSE PATH TO CLASSES COULD BE CHANGED
sqla_db_used = False
-target = None
+target_hive = None
if hive_jdbc_driver == "com.microsoft.sqlserver.jdbc.SQLServerDriver":
jdbc_jar_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
elif hive_jdbc_driver == "com.mysql.jdbc.Driver":
@@ -219,11 +219,11 @@ if not hive_use_existing_db:
downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
-target = format("{hive_lib}/{jdbc_jar_name}")
+target_hive = format("{hive_lib}/{jdbc_jar_name}")
driver_curl_source = format("{jdk_location}/{jdbc_jar_name}")
if not (stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted_major)):
- source_jdbc_file = target
+ source_jdbc_file = target_hive
else:
# normally, the JDBC driver would be referenced by <stack-root>/current/.../foo.jar
# but in RU if <stack-selector-tool> is called and the restart fails, then this means that current pointer
@@ -248,13 +248,21 @@ if sqla_db_used:
downloaded_custom_connector = format("{tmp_dir}/sqla-client-jdbc.tar.gz")
libs_in_hive_lib = format("{jdbc_libs_dir}/*")
-#common
-hive_metastore_hosts = config['clusterHostInfo']['hive_metastore_host']
-hive_metastore_host = hive_metastore_hosts[0]
-hive_metastore_port = get_port_from_url(config['configurations']['hive-site']['hive.metastore.uris']) #"9083"
+
+# Start, Common Hosts and Ports
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
-hive_server_host = config['clusterHostInfo']['hive_server_host'][0]
-hive_server_hosts = config['clusterHostInfo']['hive_server_host']
+
+hive_metastore_hosts = default('/clusterHostInfo/hive_metastore_host', [])
+hive_metastore_host = hive_metastore_hosts[0] if len(hive_metastore_hosts) > 0 else None
+hive_metastore_port = get_port_from_url(config['configurations']['hive-site']['hive.metastore.uris'])
+
+hive_server_hosts = default("/clusterHostInfo/hive_server_host", [])
+hive_server_host = hive_server_hosts[0] if len(hive_server_hosts) > 0 else None
+
+hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
+hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None
+# End, Common Hosts and Ports
+
hive_transport_mode = config['configurations']['hive-site']['hive.server2.transport.mode']
if hive_transport_mode.lower() == "http":
@@ -295,8 +303,13 @@ hive_interactive_pid = status_params.hive_interactive_pid
#Default conf dir for client
hive_conf_dirs_list = [hive_client_conf_dir]
-if hostname in hive_metastore_hosts or hostname in hive_server_hosts:
+# These are the folders to which the configs will be written to.
+if status_params.role == "HIVE_METASTORE" and hive_metastore_hosts is not None and hostname in hive_metastore_hosts:
+ hive_conf_dirs_list.append(hive_server_conf_dir)
+elif status_params.role == "HIVE_SERVER" and hive_server_hosts is not None and hostname in hive_server_host:
hive_conf_dirs_list.append(hive_server_conf_dir)
+elif status_params.role == "HIVE_SERVER_INTERACTIVE" and hive_server_interactive_hosts is not None and hostname in hive_server_interactive_hosts:
+ hive_conf_dirs_list.append(status_params.hive_server_interactive_conf_dir)
#Starting hiveserver2
@@ -317,7 +330,7 @@ artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
# Need this for yarn.nodemanager.recovery.dir in yarn-site
yarn_log_dir_prefix = config['configurations']['yarn-env']['yarn_log_dir_prefix']
-target = format("{hive_lib}/{jdbc_jar_name}")
+target_hive = format("{hive_lib}/{jdbc_jar_name}")
target_hive_interactive = format("{hive_interactive_lib}/{jdbc_jar_name}")
jars_in_hive_lib = format("{hive_lib}/*.jar")
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
index feca9f1..75507a3 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
@@ -42,6 +42,9 @@ SERVER_ROLE_DIRECTORY_MAP = {
'HIVE_SERVER_INTERACTIVE' : 'hive-server2-hive2'
}
+
+# Either HIVE_METASTORE, HIVE_SERVER, WEBHCAT_SERVER, HIVE_CLIENT, HCAT, HIVE_SERVER_INTERACTIVE
+role = default("/role", None)
component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "HIVE_CLIENT")
component_directory_interactive = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "HIVE_SERVER_INTERACTIVE")
@@ -90,9 +93,8 @@ else:
hive_client_conf_dir = "/etc/hive/conf"
hive_server_conf_dir = "/etc/hive/conf.server"
-
hive_server_interactive_conf_dir = "/etc/hive2/conf.server"
-
+
if stack_version_formatted_major and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted_major):
webhcat_conf_dir = format("{stack_root}/current/hive-webhcat/conf")
hive_conf_dir = format("{stack_root}/current/{component_directory}/conf")
@@ -109,7 +111,7 @@ else:
# if stack version supports hive serve interactive
if stack_version_formatted_major and check_stack_feature(StackFeature.HIVE_SERVER_INTERACTIVE, stack_version_formatted_major):
- hive_server_interactive_conf_dir = format("{stack_root}/current/{component_directory_interactive}/conf")
+ hive_server_interactive_conf_dir = format("{stack_root}/current/{component_directory_interactive}/conf/conf.server")
hive_config_dir = hive_client_conf_dir
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/main/resources/scripts/Ambaripreupload.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
index 941e762..99c6b46 100644
--- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py
+++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
@@ -341,8 +341,7 @@ with Environment() as env:
copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/tez/lib/tez.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/tez/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/hive/hive.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/hive/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
- # These 2 are needed by Hive Interactive
- copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/hive2/hive.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/hive2/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
+ # Needed by Hive Interactive
copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/tez_hive2/lib/tez.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/tez_hive2/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/pig/pig.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/pig/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
index ac4f6fc..845459a 100644
--- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
+++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
@@ -56,7 +56,8 @@ class TestHiveServerInteractive(RMFTestCase):
@patch("os.path.isfile")
@patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
@patch("socket.socket")
- def test_start_default(self, socket_mock, copy_to_hfds_mock, is_file_mock):
+ @patch("time.sleep")
+ def test_start_default(self, sleep_mock, socket_mock, copy_to_hfds_mock, is_file_mock):
self.maxDiff = None
copy_to_hfds_mock.return_value = False
s = socket_mock.return_value
@@ -77,21 +78,21 @@ class TestHiveServerInteractive(RMFTestCase):
user='hive'
)
self.assertResourceCalled('Execute',
- 'hive --config /etc/hive2/conf.server --service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020 OK.',
- environment={'PATH': '/usr/bin'},
+ 'hive --config /usr/hdp/current/hive-server2-hive2/conf/conf.server --service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020 OK.',
+ environment={'PATH': '/usr/hdp/current/hadoop-client/bin'},
user='hive'
)
self.assertResourceCalled('Execute',
- '/tmp/start_hiveserver2_interactive_script /var/run/hive/hive-server2-interactive.out /var/log/hive/hive-server2-interactive.err /var/run/hive/hive-interactive.pid /etc/hive2/conf.server /var/log/hive',
- environment={'HADOOP_HOME': '/usr',
+ '/tmp/start_hiveserver2_interactive_script /var/run/hive/hive-server2-interactive.out /var/log/hive/hive-server2-interactive.err /var/run/hive/hive-interactive.pid /usr/hdp/current/hive-server2-hive2/conf/conf.server /var/log/hive',
+ environment={'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
'HIVE_BIN': 'hive2',
'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
not_if="ls /var/run/hive/hive-interactive.pid >/dev/null 2>&1 && ps -p 123 >/dev/null 2>&1",
user='hive',
- path=['/bin:/usr/lib/hive/bin:/usr/bin'],
+ path=['/bin:/usr/hdp/current/hive-server2-hive2/bin:/usr/hdp/current/hadoop-client/bin'],
)
self.assertResourceCalled('Execute',
- '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/lib/hive/lib//mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
+ '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
tries=5,
try_sleep=10
@@ -133,36 +134,35 @@ class TestHiveServerInteractive(RMFTestCase):
self.assertResourceCalled('Directory', '/etc/hive2',
mode=0755,
)
- self.assertResourceCalled('Directory', '/etc/hive2/conf.server',
+ self.assertResourceCalled('Directory', '/usr/hdp/current/hive-server2-hive2/conf',
owner='hive',
group='hadoop',
create_parents=True,
- )
-
+ )
self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
group='hadoop',
- conf_dir='/etc/hive2/conf.server',
+ conf_dir='/usr/hdp/current/hive-server2-hive2/conf',
mode=0644,
configuration_attributes={u'final': {u'mapred.healthChecker.script.path': u'true',
u'mapreduce.jobtracker.staging.root.dir': u'true'}},
owner='hive',
configurations=self.getConfig()['configurations']['mapred-site'],
)
- self.assertResourceCalled('File', '/etc/hive2/conf.server/hive-default.xml.template',
+ self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/hive-default.xml.template',
owner='hive',
group='hadoop',
)
- self.assertResourceCalled('File', '/etc/hive2/conf.server/hive-env.sh.template',
+ self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/hive-env.sh.template',
owner='hive',
group='hadoop',
)
- self.assertResourceCalled('File', '/etc/hive2/conf.server/hive-exec-log4j.properties',
+ self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/hive-exec-log4j.properties',
content='log4jproperties\nline2',
owner='hive',
group='hadoop',
mode=0644,
)
- self.assertResourceCalled('File', '/etc/hive2/conf.server/hive-log4j.properties',
+ self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/hive-log4j.properties',
content='log4jproperties\nline2',
owner='hive',
group='hadoop',
@@ -173,9 +173,57 @@ class TestHiveServerInteractive(RMFTestCase):
hive_site_conf.update(self.getConfig()['configurations']['hive-interactive-site'])
del hive_site_conf['hive.enforce.bucketing']
del hive_site_conf['hive.enforce.sorting']
+
+ mapred_site_conf = {}
+ mapred_site_conf.update(self.getConfig()['configurations']['mapred-site'])
+
+ self.assertResourceCalled("Directory", "/usr/hdp/current/hive-server2-hive2/conf/conf.server",
+ owner=u"hive",
+ group=u"hadoop",
+ create_parents=True)
+
+ self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
+ group='hadoop',
+ conf_dir='/usr/hdp/current/hive-server2-hive2/conf/conf.server',
+ mode=0644,
+ configuration_attributes={u'final': {u'mapred.healthChecker.script.path': u'true',
+ u'mapreduce.jobtracker.staging.root.dir': u'true'}},
+ owner='hive',
+ configurations=mapred_site_conf,
+ )
+
+ self.assertResourceCalled("File", "/usr/hdp/current/hive-server2-hive2/conf/conf.server/hive-default.xml.template",
+ owner=u"hive",
+ group=u"hadoop")
+
+ self.assertResourceCalled("File", "/usr/hdp/current/hive-server2-hive2/conf/conf.server/hive-env.sh.template",
+ owner=u"hive",
+ group=u"hadoop")
+
+ self.assertResourceCalled("File", "/usr/hdp/current/hive-server2-hive2/conf/conf.server/hive-exec-log4j.properties",
+ content=u'log4jproperties\nline2',
+ owner=u'hive',
+ group=u'hadoop',
+ mode=420)
+ self.assertResourceCalled("File", "/usr/hdp/current/hive-server2-hive2/conf/conf.server/hive-log4j.properties",
+ content=u'log4jproperties\nline2',
+ owner=u'hive',
+ group=u'hadoop',
+ mode=420)
+
+ self.assertResourceCalled('XmlConfig', 'hive-site.xml',
+ group='hadoop',
+ conf_dir='/usr/hdp/current/hive-server2-hive2/conf/conf.server',
+ mode=0644,
+ configuration_attributes={u'final': {u'hive.optimize.bucketmapjoin.sortedmerge': u'true',
+ u'javax.jdo.option.ConnectionDriverName': u'true',
+ u'javax.jdo.option.ConnectionPassword': u'true'}},
+ owner='hive',
+ configurations=hive_site_conf,
+ )
self.assertResourceCalled('XmlConfig', 'hive-site.xml',
group='hadoop',
- conf_dir='/etc/hive2/conf.server',
+ conf_dir='/usr/hdp/current/hive-server2-hive2/conf',
mode=0644,
configuration_attributes={u'final': {u'hive.optimize.bucketmapjoin.sortedmerge': u'true',
u'javax.jdo.option.ConnectionDriverName': u'true',
@@ -191,7 +239,7 @@ class TestHiveServerInteractive(RMFTestCase):
owner='tez',
configurations=self.getConfig()['configurations']['tez-interactive-site'],
)
- self.assertResourceCalled('File', '/etc/hive2/conf.server/hive-env.sh',
+ self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/conf.server/hive-env.sh',
content=InlineTemplate(self.getConfig()['configurations']['hive-interactive-env']['content']),
owner='hive',
group='hadoop',
@@ -210,11 +258,11 @@ class TestHiveServerInteractive(RMFTestCase):
self.assertResourceCalled('Execute', ('cp',
'--remove-destination',
'/usr/share/java/mysql-connector-java.jar',
- '/usr/lib/hive/lib//mysql-connector-java.jar'),
+ '/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar'),
path=['/bin', '/usr/bin/'],
sudo=True,
)
- self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar',
+ self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar',
mode=0644,
)
self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
http://git-wip-us.apache.org/repos/asf/ambari/blob/a02ae60d/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
index f9c31dc..d077c35 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json
@@ -12,7 +12,7 @@
"ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
"repo_info": "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
"jce_name": "UnlimitedJCEPolicyJDK7.zip",
- "stack_version": "2.0",
+ "stack_version": "2.5",
"stack_name": "HDP",
"ambari_db_rca_driver": "org.postgresql.Driver",
"jdk_name": "jdk-7u67-linux-x64.tar.gz",
@@ -30,7 +30,7 @@
"commandType": "EXECUTION_COMMAND",
"roleParams": {},
"serviceName": "HIVE",
- "role": "HIVE_SERVER",
+ "role": "HIVE_SERVER_INTERACTIVE",
"commandParams": {
"command_timeout": "300",
"service_package_folder": "OOZIE",
@@ -1212,7 +1212,7 @@
"c6402.ambari.apache.org"
],
"hive_server_interactive_hosts": [
- "c6402.ambari.apache.org"
+ "c6401.ambari.apache.org"
],
"falcon_server_hosts": [
"c6402.ambari.apache.org"