You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2014/09/30 22:48:26 UTC
[20/35] git commit: AMBARI-7553. Ambari needs to support
WebHCat/Sqoop integration config.(vbrodetskyi)
AMBARI-7553. Ambari needs to support WebHCat/Sqoop integration config.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba530c8d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba530c8d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba530c8d
Branch: refs/heads/branch-alerts-dev
Commit: ba530c8d3c7f10f40069c0b8734dbd1cbca557c6
Parents: bbed852
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue Sep 30 09:34:08 2014 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue Sep 30 09:34:08 2014 +0300
----------------------------------------------------------------------
.../services/HIVE/package/scripts/params.py | 2 ++
.../services/HIVE/package/scripts/webhcat.py | 32 ++++++++++++++------
.../HIVE/configuration/webhcat-site.xml | 11 +++++++
.../stacks/2.0.6/HIVE/test_webhcat_server.py | 20 ++++++++++++
4 files changed, 56 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba530c8d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
index e64ffa3..7e75433 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
@@ -43,6 +43,7 @@ if rpm_version is not None:
tez_tar_file = "/usr/hdp/current/tez/lib/tez*.tar.gz"
pig_tar_file = '/usr/hdp/current/pig/pig.tar.gz'
hive_tar_file = '/usr/hdp/current/hive/hive.tar.gz'
+ sqoop_tar_file = '/usr/hdp/current/sqoop/sqoop*.tar.gz'
hcat_lib = '/usr/hdp/current/hive/hive-hcatalog/share/hcatalog'
webhcat_bin_dir = '/usr/hdp/current/hive-hcatalog/sbin'
@@ -58,6 +59,7 @@ else:
tez_tar_file = "/usr/lib/tez/tez*.tar.gz"
pig_tar_file = '/usr/share/HDP-webhcat/pig.tar.gz'
hive_tar_file = '/usr/share/HDP-webhcat/hive.tar.gz'
+ sqoop_tar_file = '/usr/share/HDP-webhcat/sqoop*.tar.gz'
if str(hdp_stack_version).startswith('2.0'):
hcat_lib = '/usr/lib/hcatalog/share/hcatalog'
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba530c8d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
index 6ce47f5..b034cbc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
@@ -20,6 +20,8 @@ Ambari Agent
"""
from resource_management import *
import sys
+import os.path
+import glob
def webhcat():
@@ -95,15 +97,16 @@ def webhcat():
hadoop_conf_dir=params.hadoop_conf_dir
)
- CopyFromLocal(params.pig_tar_file,
- owner=params.webhcat_user,
- mode=0755,
- dest_dir=params.webhcat_apps_dir,
- kinnit_if_needed=kinit_if_needed,
- hdfs_user=params.hdfs_user,
- hadoop_bin_dir=params.hadoop_bin_dir,
- hadoop_conf_dir=params.hadoop_conf_dir
- )
+ if (os.path.isfile(params.pig_tar_file)):
+ CopyFromLocal(params.pig_tar_file,
+ owner=params.webhcat_user,
+ mode=0755,
+ dest_dir=params.webhcat_apps_dir,
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user,
+ hadoop_bin_dir=params.hadoop_bin_dir,
+ hadoop_conf_dir=params.hadoop_conf_dir
+ )
CopyFromLocal(params.hive_tar_file,
owner=params.webhcat_user,
@@ -114,3 +117,14 @@ def webhcat():
hadoop_bin_dir=params.hadoop_bin_dir,
hadoop_conf_dir=params.hadoop_conf_dir
)
+
+ if (len(glob.glob(params.sqoop_tar_file)) > 0):
+ CopyFromLocal(params.sqoop_tar_file,
+ owner=params.webhcat_user,
+ mode=0755,
+ dest_dir=params.webhcat_apps_dir,
+ kinnit_if_needed=kinit_if_needed,
+ hdfs_user=params.hdfs_user,
+ hadoop_bin_dir=params.hadoop_bin_dir,
+ hadoop_conf_dir=params.hadoop_conf_dir
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba530c8d/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
index bdecb67..f0d6a84 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
@@ -49,5 +49,16 @@ limitations under the License.
<description>The path to the hcatalog executable.</description>
</property>
+ <property>
+ <name>templeton.sqoop.archive</name>
+ <value>hdfs:///apps/webhcat/sqoop.tar.gz</value>
+ <description>The path to the Sqoop archive in HDFS.</description>
+ </property>
+
+ <property>
+ <name>templeton.sqoop.path</name>
+ <value>sqoop.tar.gz/sqoop/bin/sqoop</value>
+ <description>The path to the Sqoop executable.</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba530c8d/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
index 31588b8..5f92a2d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
@@ -20,6 +20,8 @@ limitations under the License.
from mock.mock import MagicMock, patch
from stacks.utils.RMFTestCase import *
+@patch("os.path.isfile", new = MagicMock(return_value=True))
+@patch("glob.glob", new = MagicMock(return_value=["one", "two"]))
class TestWebHCatServer(RMFTestCase):
def test_configure_default(self):
@@ -183,6 +185,15 @@ class TestWebHCatServer(RMFTestCase):
hadoop_conf_dir='/etc/hadoop/conf',
hdfs_user='hdfs'
)
+ self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/sqoop*.tar.gz',
+ owner='hcat',
+ mode=0755,
+ dest_dir='/apps/webhcat',
+ kinnit_if_needed='',
+ hadoop_bin_dir='/usr/bin',
+ hadoop_conf_dir='/etc/hadoop/conf',
+ hdfs_user='hdfs'
+ )
def assert_configure_secured(self):
self.assertResourceCalled('HdfsDirectory', '/apps/webhcat',
@@ -275,4 +286,13 @@ class TestWebHCatServer(RMFTestCase):
hadoop_conf_dir='/etc/hadoop/conf',
hadoop_bin_dir='/usr/bin',
hdfs_user='hdfs'
+ )
+ self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/sqoop*.tar.gz',
+ owner='hcat',
+ mode=0755,
+ dest_dir='/apps/webhcat',
+ kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs;',
+ hadoop_conf_dir='/etc/hadoop/conf',
+ hadoop_bin_dir='/usr/bin',
+ hdfs_user='hdfs'
)
\ No newline at end of file