You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by st...@apache.org on 2017/01/20 06:03:04 UTC
ambari git commit: AMBARI-19629. Move schemaTool invocation out of
configure() to hive_metastore start(). (Attila Doroszlai via stoader)
Repository: ambari
Updated Branches:
refs/heads/trunk 53b3ecfd9 -> 6ba9bd0a7
AMBARI-19629. Move schemaTool invocation out of configure() to hive_metastore start(). (Attila Doroszlai via stoader)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6ba9bd0a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6ba9bd0a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6ba9bd0a
Branch: refs/heads/trunk
Commit: 6ba9bd0a7d5534838a7c1679675cf1848806990f
Parents: 53b3ecf
Author: Attila Doroszlai <ad...@hortonworks.com>
Authored: Fri Jan 20 06:55:14 2017 +0100
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Fri Jan 20 06:55:14 2017 +0100
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/package/scripts/hive.py | 57 ++++++++++----------
.../package/scripts/hive_metastore.py | 5 +-
.../stacks/2.0.6/HIVE/test_hive_metastore.py | 9 ++--
.../stacks/2.1/HIVE/test_hive_metastore.py | 14 ++---
4 files changed, 46 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba9bd0a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index 16273c7..2a4fd30 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -121,7 +121,7 @@ def hive(name=None):
if name == 'hiveserver2':
setup_hiveserver2()
if name == 'metastore':
- setup_metastore() # schematool work
+ setup_metastore()
def setup_hiveserver2():
import params
@@ -299,32 +299,35 @@ def setup_metastore():
mode=0755,
content=StaticFile('startMetastore.sh')
)
- if params.init_metastore_schema:
- create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
- "{hive_schematool_bin}/schematool -initSchema "
- "-dbType {hive_metastore_db_type} "
- "-userName {hive_metastore_user_name} "
- "-passWord {hive_metastore_user_passwd!p} -verbose")
-
- check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
- "{hive_schematool_bin}/schematool -info "
- "-dbType {hive_metastore_db_type} "
- "-userName {hive_metastore_user_name} "
- "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user)
-
- # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords.
- # Fixing it with the hack below:
- quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd))
- if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \
- or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"':
- quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1]
- Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace(
- format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING))
-
- Execute(create_schema_cmd,
- not_if = check_schema_created_cmd,
- user = params.hive_user
- )
+
+def create_metastore_schema():
+ import params
+
+ create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
+ "{hive_schematool_bin}/schematool -initSchema "
+ "-dbType {hive_metastore_db_type} "
+ "-userName {hive_metastore_user_name} "
+ "-passWord {hive_metastore_user_passwd!p} -verbose")
+
+ check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
+ "{hive_schematool_bin}/schematool -info "
+ "-dbType {hive_metastore_db_type} "
+ "-userName {hive_metastore_user_name} "
+ "-passWord {hive_metastore_user_passwd!p} -verbose"), params.hive_user)
+
+ # HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords.
+ # Fixing it with the hack below:
+ quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd))
+ if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \
+ or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"':
+ quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1]
+ Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace(
+ format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING))
+
+ Execute(create_schema_cmd,
+ not_if = check_schema_created_cmd,
+ user = params.hive_user
+ )
"""
Writes configuration files required by Hive.
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba9bd0a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
index d1ecd65..8b69e45 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_metastore.py
@@ -36,8 +36,7 @@ from resource_management.libraries.functions.security_commons import validate_se
from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
from resource_management.core.resources.system import File
-from hive import hive
-from hive import jdbc_connector
+from hive import create_metastore_schema, hive, jdbc_connector
from hive_service import hive_service
from ambari_commons.os_family_impl import OsFamilyImpl
from ambari_commons import OSConst
@@ -57,6 +56,8 @@ class HiveMetastore(Script):
# writing configurations on start required for securtity
self.configure(env)
+ if params.init_metastore_schema:
+ create_metastore_schema() # execute without config lock
hive_service('metastore', action='start', upgrade_type=upgrade_type)
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba9bd0a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
index 0a98894..9c04acc 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py
@@ -36,6 +36,7 @@ class TestHiveMetastore(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_default()
+ self.assertNoMoreResources()
def test_start_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
@@ -47,6 +48,7 @@ class TestHiveMetastore(RMFTestCase):
)
self.assert_configure_default()
+ self.assert_init_schema()
self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive',
environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
'HIVE_BIN': 'hive',
@@ -107,6 +109,7 @@ class TestHiveMetastore(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_secured()
+ self.assert_init_schema()
self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive',
environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
'HIVE_BIN': 'hive',
@@ -264,6 +267,8 @@ class TestHiveMetastore(RMFTestCase):
content = StaticFile('startMetastore.sh'),
mode = 0755,
)
+
+ def assert_init_schema(self):
self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord \'!`"\'"\'"\' 1\' -verbose',
not_if = 'ambari-sudo.sh su hive -l -s /bin/bash -c \'[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord \'"\'"\'!`"\'"\'"\'"\'"\'"\'"\'"\'"\' 1\'"\'"\' -verbose\'',
user = 'hive',
@@ -387,10 +392,6 @@ class TestHiveMetastore(RMFTestCase):
content = StaticFile('startMetastore.sh'),
mode = 0755,
)
- self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord \'!`"\'"\'"\' 1\' -verbose',
- not_if = 'ambari-sudo.sh su hive -l -s /bin/bash -c \'[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord \'"\'"\'!`"\'"\'"\'"\'"\'"\'"\'"\'"\' 1\'"\'"\' -verbose\'',
- user = 'hive',
- )
@patch("resource_management.core.shell.call")
@patch("resource_management.libraries.functions.get_stack_version")
http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba9bd0a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
index f9e910b..a159cd0 100644
--- a/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
+++ b/ambari-server/src/test/python/stacks/2.1/HIVE/test_hive_metastore.py
@@ -42,6 +42,7 @@ class TestHiveMetastore(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_default()
+ self.assertNoMoreResources()
def test_start_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_metastore.py",
@@ -53,6 +54,7 @@ class TestHiveMetastore(RMFTestCase):
)
self.assert_configure_default()
+ self.assert_init_schema('aaa')
self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive',
environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
@@ -83,6 +85,7 @@ class TestHiveMetastore(RMFTestCase):
)
self.assert_configure_default()
+ self.assert_init_schema('aaa')
self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive',
environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
@@ -146,6 +149,7 @@ class TestHiveMetastore(RMFTestCase):
)
self.assert_configure_secured()
+ self.assert_init_schema('asd')
self.assertResourceCalled('Execute', '/tmp/start_metastore_script /var/log/hive/hive.out /var/log/hive/hive.err /var/run/hive/hive.pid /etc/hive/conf.server /var/log/hive',
environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
'HIVE_BIN': 'hive',
@@ -287,8 +291,10 @@ class TestHiveMetastore(RMFTestCase):
content = StaticFile('startMetastore.sh'),
mode = 0755,
)
- self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord aaa -verbose',
- not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord aaa -verbose'",
+
+ def assert_init_schema(self, password):
+ self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord {password} -verbose'.format(password = password),
+ not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord {password} -verbose'".format(password = password),
user = 'hive',
)
@@ -399,10 +405,6 @@ class TestHiveMetastore(RMFTestCase):
content = StaticFile('startMetastore.sh'),
mode = 0755,
)
- self.assertResourceCalled('Execute', 'export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -initSchema -dbType mysql -userName hive -passWord asd -verbose',
- not_if = "ambari-sudo.sh su hive -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]export HIVE_CONF_DIR=/etc/hive/conf.server ; /usr/hdp/current/hive-server2/bin/schematool -info -dbType mysql -userName hive -passWord asd -verbose'",
- user = 'hive',
- )
@patch("resource_management.libraries.functions.security_commons.build_expectations")
@patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")