You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2014/10/29 18:15:13 UTC
git commit: AMBARI-8016 Cannot run hive tez query using HDP 2.1 stack
(dsen)
Repository: ambari
Updated Branches:
refs/heads/branch-1.7.0 1b3cfd6c4 -> ee9be26cc
AMBARI-8016 Cannot run hive tez query using HDP 2.1 stack (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ee9be26c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ee9be26c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ee9be26c
Branch: refs/heads/branch-1.7.0
Commit: ee9be26cc7168b607242cab8f0956ff5e21cf349
Parents: 1b3cfd6
Author: Dmytro Sen <ds...@apache.org>
Authored: Wed Oct 29 19:14:34 2014 +0200
Committer: Dmytro Sen <ds...@apache.org>
Committed: Wed Oct 29 19:14:34 2014 +0200
----------------------------------------------------------------------
.../HIVE/package/scripts/hive_server.py | 4 +-
.../HIVE/package/scripts/install_jars.py | 107 +++++++++++++++++++
.../services/HIVE/package/scripts/params.py | 8 ++
.../stacks/2.0.6/HIVE/test_hive_server.py | 53 +++++++++
4 files changed, 171 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ee9be26c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
index 4e55cfb..1110544 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
@@ -22,6 +22,7 @@ from resource_management import *
from hive import hive
from hive_service import hive_service
from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
+from install_jars import install_tez_jars
class HiveServer(Script):
@@ -32,7 +33,8 @@ class HiveServer(Script):
def configure(self, env):
import params
env.set_params(params)
-
+ if not params.stack_is_hdp22_or_further:
+ install_tez_jars()
hive(name='hiveserver2')
def start(self, env):
http://git-wip-us.apache.org/repos/asf/ambari/blob/ee9be26c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
new file mode 100644
index 0000000..a18ca72
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import os
+
+def install_tez_jars():
+ import params
+
+ destination_hdfs_dirs = get_tez_hdfs_dir_paths(params.tez_lib_uris)
+
+ # If tez libraries are to be stored in hdfs
+ if destination_hdfs_dirs:
+ for hdfs_dir in destination_hdfs_dirs:
+ params.HdfsDirectory(hdfs_dir,
+ action="create_delayed",
+ owner=params.tez_user,
+ mode=0755
+ )
+ pass
+ params.HdfsDirectory(None, action="create")
+
+ if params.security_enabled:
+ kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
+ else:
+ kinit_if_needed = ""
+
+ if kinit_if_needed:
+ Execute(kinit_if_needed,
+ user=params.tez_user,
+ path='/bin'
+ )
+ pass
+
+ app_dir_path = None
+ lib_dir_path = None
+
+ if len(destination_hdfs_dirs) > 0:
+ for path in destination_hdfs_dirs:
+ if 'lib' in path:
+ lib_dir_path = path
+ else:
+ app_dir_path = path
+ pass
+ pass
+ pass
+
+ if app_dir_path:
+ for scr_file, dest_file in params.app_dir_files.iteritems():
+ CopyFromLocal(scr_file,
+ mode=0755,
+ owner=params.tez_user,
+ dest_dir=app_dir_path,
+ dest_file=dest_file,
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user,
+ hadoop_bin_dir=params.hadoop_bin_dir,
+ hadoop_conf_dir=params.hadoop_conf_dir
+ )
+
+ if lib_dir_path:
+ CopyFromLocal(params.tez_local_lib_jars,
+ mode=0755,
+ owner=params.tez_user,
+ dest_dir=lib_dir_path,
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user,
+ hadoop_bin_dir=params.hadoop_bin_dir,
+ hadoop_conf_dir=params.hadoop_conf_dir
+ )
+ pass
+
+
+def get_tez_hdfs_dir_paths(tez_lib_uris = None):
+ hdfs_path_prefix = 'hdfs://'
+ lib_dir_paths = []
+ if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1:
+ dir_paths = tez_lib_uris.split(',')
+ for path in dir_paths:
+ if not "tez.tar.gz" in path:
+ lib_dir_path = path.replace(hdfs_path_prefix, '')
+ lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
+ lib_dir_paths.append(lib_dir_path)
+ else:
+ lib_dir_path = path.replace(hdfs_path_prefix, '')
+ lib_dir_paths.append(os.path.dirname(lib_dir_path))
+ pass
+ pass
+
+ return lib_dir_paths
http://git-wip-us.apache.org/repos/asf/ambari/blob/ee9be26c/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
index 2bfbdc1..41c5d1a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
@@ -211,6 +211,14 @@ hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_nam
# Tez-related properties
tez_user = config['configurations']['tez-env']['tez_user']
+# Tez jars
+tez_local_api_jars = '/usr/lib/tez/tez*.jar'
+tez_local_lib_jars = '/usr/lib/tez/lib/*.jar'
+app_dir_files = {tez_local_api_jars:None}
+
+# Tez libraries
+tez_lib_uris = default("/configurations/tez-site/tez.lib.uris", None)
+
if System.get_instance().os_family == "ubuntu":
mysql_configname = '/etc/mysql/my.cnf'
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/ee9be26c/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
index 0abc438..f28e821 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py
@@ -137,6 +137,59 @@ class TestHiveServer(RMFTestCase):
self.assertFalse(socket_mock.called)
def assert_configure_default(self):
+ self.assertResourceCalled('HdfsDirectory', '/apps/tez/',
+ action = ['create_delayed'],
+ mode = 0755,
+ owner = 'tez',
+ security_enabled = False,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = 'hdfs',
+ bin_dir = '/usr/bin',
+ kinit_path_local = "/usr/bin/kinit"
+ )
+
+ self.assertResourceCalled('HdfsDirectory', '/apps/tez/lib/',
+ action = ['create_delayed'],
+ mode = 0755,
+ owner = 'tez',
+ security_enabled = False,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = 'hdfs',
+ bin_dir = '/usr/bin',
+ kinit_path_local = "/usr/bin/kinit"
+ )
+ self.assertResourceCalled('HdfsDirectory', None,
+ security_enabled = False,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = 'hdfs',
+ kinit_path_local = '/usr/bin/kinit',
+ bin_dir = '/usr/bin',
+ action = ['create']
+ )
+
+ self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/tez*.jar',
+ mode=0755,
+ owner='tez',
+ dest_dir='/apps/tez/',
+ kinnit_if_needed='',
+ hadoop_conf_dir='/etc/hadoop/conf',
+ hadoop_bin_dir='/usr/bin',
+ hdfs_user='hdfs',
+ dest_file=None
+ )
+
+ self.assertResourceCalled('CopyFromLocal', '/usr/lib/tez/lib/*.jar',
+ mode=0755,
+ owner='tez',
+ dest_dir='/apps/tez/lib/',
+ kinnit_if_needed='',
+ hadoop_bin_dir='/usr/bin',
+ hadoop_conf_dir='/etc/hadoop/conf',
+ hdfs_user='hdfs'
+ )
self.assertResourceCalled('HdfsDirectory', '/apps/hive/warehouse',
security_enabled = False,
keytab = UnknownConfigurationMock(),