You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2016/02/03 01:19:51 UTC
ambari git commit: AMBARI-14816: Run a service check on PXF service
for Hive (lavjain via jaoki)
Repository: ambari
Updated Branches:
refs/heads/trunk 6fb00585c -> 7632b7bb2
AMBARI-14816: Run a service check on PXF service for Hive (lavjain via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7632b7bb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7632b7bb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7632b7bb
Branch: refs/heads/trunk
Commit: 7632b7bb2454a67015308da25c588dd0bf4aafe2
Parents: 6fb0058
Author: Jun Aoki <ja...@apache.org>
Authored: Tue Feb 2 16:19:24 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Tue Feb 2 16:19:24 2016 -0800
----------------------------------------------------------------------
.../PXF/3.0.0/package/scripts/params.py | 6 +-
.../PXF/3.0.0/package/scripts/pxf_constants.py | 1 +
.../PXF/3.0.0/package/scripts/service_check.py | 101 +++++++++++++------
3 files changed, 76 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7632b7bb/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
index 36dc7c1..7749de7 100644
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
@@ -16,7 +16,6 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
-import os
import functools
from resource_management import Script
@@ -35,6 +34,7 @@ pxf_group = pxf_user
hdfs_superuser = config['configurations']['hadoop-env']['hdfs_user']
hdfs_superuser_group = config["configurations"]["hdfs-site"]["dfs.permissions.superusergroup"]
user_group = config["configurations"]["cluster-env"]["user_group"]
+hive_user = default('configurations/hive-env/hive_user', None)
tomcat_group = "tomcat"
# Directories
@@ -54,11 +54,13 @@ realm_name = config['configurations']['kerberos-env']['realm']
#HBase
is_hbase_installed = default("/clusterHostInfo/hbase_master_hosts", None) is not None
+#Hive
+is_hive_installed = default("/clusterHostInfo/hive_server_host", None) is not None
+
# HDFS
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
-security_enabled = config['configurations']['cluster-env']['security_enabled']
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
http://git-wip-us.apache.org/repos/asf/ambari/blob/7632b7bb/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
index 3138379..9d93a38 100644
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
@@ -24,3 +24,4 @@ pxf_hdfs_test_dir = "/pxf_hdfs_smoke_test"
pxf_hdfs_read_test_file = pxf_hdfs_test_dir + "/pxf_smoke_test_read_data"
pxf_hdfs_write_test_file = pxf_hdfs_test_dir + "/pxf_smoke_test_write_data"
pxf_hbase_test_table = "pxf_hbase_smoke_test_table"
+pxf_hive_test_table = "pxf_hive_smoke_test_table"
http://git-wip-us.apache.org/repos/asf/ambari/blob/7632b7bb/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
index 72b59e4..064be04 100644
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
@@ -24,8 +24,6 @@ from resource_management.core.resources.system import Execute
from pxf_utils import makeHTTPCall, runLocalCmd
import pxf_constants
-import sys
-
class PXFServiceCheck(Script):
"""
Runs a set of simple PXF tests to verify if the service has been setup correctly
@@ -51,10 +49,11 @@ class PXFServiceCheck(Script):
import params
self.pxf_version = self.__get_pxf_protocol_version()
try:
- self.cleanup_test_data()
self.run_hdfs_tests()
if params.is_hbase_installed:
self.run_hbase_tests()
+ if params.is_hive_installed:
+ self.run_hive_tests()
except:
msg = "PXF service check failed"
Logger.error(msg)
@@ -75,6 +74,8 @@ class PXFServiceCheck(Script):
self.__cleanup_hdfs_data()
if params.is_hbase_installed:
self.__cleanup_hbase_data()
+ if params.is_hive_installed:
+ self.__cleanup_hive_data()
def __get_pxf_protocol_version(self):
@@ -98,17 +99,33 @@ class PXFServiceCheck(Script):
raise Fail(msg)
+ def __check_pxf_read(self, headers):
+ """
+ Performs a generic PXF read
+ """
+ url = self.base_url + self.pxf_version + "/Fragmenter/getFragments?path="
+ try:
+ response = makeHTTPCall(url, headers)
+ if not "PXFFragments" in response:
+ Logger.error("Unable to find PXFFragments in the response")
+ raise
+ except:
+ msg = "PXF data read failed"
+ raise Fail(msg)
+
+
+ # HDFS Routines
def run_hdfs_tests(self):
"""
Runs a set of PXF HDFS checks
"""
Logger.info("Running PXF HDFS checks")
- self.__check_if_client_exists("HDFS")
+ self.__check_if_client_exists("Hadoop-HDFS")
+ self.__cleanup_hdfs_data()
self.__write_hdfs_data()
self.__check_pxf_hdfs_read()
self.__check_pxf_hdfs_write()
-
def __write_hdfs_data(self):
"""
Writes some test HDFS data for the tests
@@ -127,22 +144,6 @@ class PXFServiceCheck(Script):
action="create_on_execute"
)
-
- def __check_pxf_read(self, headers):
- """
- Performs a generic PXF read
- """
- url = self.base_url + self.pxf_version + "/Fragmenter/getFragments?path="
- try:
- response = makeHTTPCall(url, headers)
- if not "PXFFragments" in response:
- Logger.error("Unable to find PXFFragments in the response")
- raise
- except:
- msg = "PXF data read failed"
- raise Fail(msg)
-
-
def __check_pxf_hdfs_read(self):
"""
Reads the test HDFS data through PXF
@@ -155,7 +156,6 @@ class PXFServiceCheck(Script):
headers.update(self.commonPXFHeaders)
self.__check_pxf_read(headers)
-
def __check_pxf_hdfs_write(self):
"""
Writes some test HDFS data through PXF
@@ -184,7 +184,6 @@ class PXFServiceCheck(Script):
msg = "PXF HDFS data write test failed"
raise Fail(msg)
-
def __cleanup_hdfs_data(self):
"""
Cleans up the test HDFS data
@@ -201,16 +200,17 @@ class PXFServiceCheck(Script):
)
+ # HBase Routines
def run_hbase_tests(self):
"""
Runs a set of PXF HBase checks
"""
Logger.info("Running PXF HBase checks")
+ self.__cleanup_hbase_data()
self.__check_if_client_exists("HBase")
self.__write_hbase_data()
self.__check_pxf_hbase_read()
-
def __write_hbase_data(self):
"""
Creates a temporary HBase table for the service checks
@@ -219,7 +219,6 @@ class PXFServiceCheck(Script):
Execute("echo \"create '" + pxf_constants.pxf_hbase_test_table + "', 'cf'\"|hbase shell", logoutput = True)
Execute("echo \"put '" + pxf_constants.pxf_hbase_test_table + "', 'row1', 'cf:a', 'value1'; put '" + pxf_constants.pxf_hbase_test_table + "', 'row1', 'cf:b', 'value2'\" | hbase shell", logoutput = True)
-
def __check_pxf_hbase_read(self):
"""
Checks reading HBase data through PXF
@@ -233,7 +232,6 @@ class PXFServiceCheck(Script):
self.__check_pxf_read(headers)
-
def __cleanup_hbase_data(self):
"""
Cleans up the test HBase data
@@ -243,6 +241,49 @@ class PXFServiceCheck(Script):
Execute("echo \"drop '" + pxf_constants.pxf_hbase_test_table + "'\"|hbase shell > /dev/null 2>&1", logoutput = True)
+ # Hive Routines
+ def run_hive_tests(self):
+ """
+ Runs a set of PXF Hive checks
+ """
+ Logger.info("Running PXF Hive checks")
+ self.__check_if_client_exists("Hive")
+ self.__cleanup_hive_data()
+ self.__write_hive_data()
+ self.__check_pxf_hive_read()
+
+ def __write_hive_data(self):
+ """
+ Creates a temporary Hive table for the service checks
+ """
+ import params
+ Logger.info("Creating temporary Hive test data")
+ cmd = "hive -e 'CREATE TABLE IF NOT EXISTS {0} (id INT); INSERT INTO {0} VALUES (1);'".format(pxf_constants.pxf_hive_test_table)
+ Execute(cmd, logoutput = True, user = params.hive_user)
+
+ def __check_pxf_hive_read(self):
+ """
+ Checks reading Hive data through PXF
+ """
+ Logger.info("Testing PXF Hive data read")
+ headers = {
+ "X-GP-DATA-DIR": pxf_constants.pxf_hive_test_table,
+ "X-GP-profile": "Hive",
+ }
+ headers.update(self.commonPXFHeaders)
+ self.__check_pxf_read(headers)
+
+ def __cleanup_hive_data(self):
+ """
+ Cleans up the test Hive data
+ """
+ import params
+ Logger.info("Cleaning up Hive test data")
+ cmd = "hive -e 'DROP TABLE IF EXISTS {0};'".format(pxf_constants.pxf_hive_test_table)
+ Execute(cmd, logoutput = True, user = params.hive_user)
+
+
+ # Package Routines
def __package_exists(self, pkg):
"""
Low level function to check if a rpm is installed
@@ -250,15 +291,15 @@ class PXFServiceCheck(Script):
if System.get_instance().os_family == "suse":
return not runLocalCmd("zypper search " + pkg)
else:
- return not runLocalCmd("yum list installed | grep -i " + pkg)
+ return not runLocalCmd("yum list installed | egrep -i ^" + pkg)
def __check_if_client_exists(self, serviceName):
Logger.info("Checking if " + serviceName + " client libraries exist")
if not self.__package_exists(serviceName):
- error_msg = serviceName + "client libraries do not exist on the PXF node"
- Logger.error(msg)
- raise Fail(msg)
+ error_msg = serviceName + " client libraries do not exist on the PXF node"
+ Logger.error(error_msg)
+ raise Fail(error_msg)
if __name__ == "__main__":