You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/10/01 13:44:07 UTC
[1/3] ambari git commit: AMBARI-13284. HostCleanup script does not
work for hosts that are manually registered (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-2.1 719bea42f -> 055e184c9
refs/heads/trunk c538a4d2c -> d2e11c868
AMBARI-13284. HostCleanup script does not work for hosts that are manually registered (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d2e11c86
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d2e11c86
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d2e11c86
Branch: refs/heads/trunk
Commit: d2e11c86805d998796a23374a9f55b898066bbee
Parents: abf1cb8
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Oct 1 14:43:57 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Oct 1 14:43:57 2015 +0300
----------------------------------------------------------------------
.../src/main/python/ambari_agent/HostCleanup.py | 28 +++++++++++++++++++-
.../custom_actions/scripts/check_host.py | 6 +++++
nohup.out | 16 +++++++++++
3 files changed, 49 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2e11c86/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index a725d43..4d3cd83 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -34,7 +34,9 @@ import ConfigParser
import optparse
import shlex
import datetime
+import tempfile
from AmbariConfig import AmbariConfig
+from ambari_agent.Constants import AGENT_TMP_DIR
from ambari_commons import OSCheck, OSConst
from ambari_commons.constants import AMBARI_SUDO_BINARY
from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl
@@ -46,6 +48,7 @@ GROUP_ERASE_CMD = "groupdel {0}"
PROC_KILL_CMD = "kill -9 {0}"
ALT_DISP_CMD = "alternatives --display {0}"
ALT_ERASE_CMD = "alternatives --remove {0} {1}"
+RUN_HOST_CHECKS_CMD = '/var/lib/ambari-agent/cache/custom_actions/scripts/check_host.py ACTIONEXECUTE {0} /var/lib/ambari-agent/cache/custom_actions {1} INFO {2}'
REPO_PATH_RHEL = "/etc/yum.repos.d"
REPO_PATH_SUSE = "/etc/zypp/repos.d/"
@@ -492,6 +495,19 @@ class HostCleanup:
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
+ def run_check_hosts(self):
+ config_json = '{"commandParams": {"check_execute_list": "*BEFORE_CLEANUP_HOST_CHECKS*"}}'
+ with tempfile.NamedTemporaryFile(delete=False) as config_json_file:
+ config_json_file.write(config_json)
+
+ with tempfile.NamedTemporaryFile(delete=False) as tmp_output_file:
+ tmp_output_file.write('{}')
+
+ run_checks_command = RUN_HOST_CHECKS_CMD.format(config_json_file.name, tmp_output_file.name, AGENT_TMP_DIR)
+ (returncode, stdoutdata, stderrdata) = self.run_os_command(run_checks_command)
+ if returncode != 0:
+ logger.warn('Failed to run host checks,\nstderr:\n ' + stderrdata + '\n\nstdout:\n' + stdoutdata)
+
# Copy file and save with file.# (timestamp)
def backup_file(filePath):
if filePath is not None and os.path.exists(filePath):
@@ -584,7 +600,17 @@ def main():
sys.exit(1)
hostcheckfile, hostcheckfileca = options.inputfiles.split(",")
-
+
+ # Manage non UI install
+ if not os.path.exists(hostcheckfileca):
+ if options.silent:
+ print 'Host Check results not found. There is no {0}. Running host checks.'.format(hostcheckfileca)
+ h.run_check_hosts()
+ else:
+ run_check_hosts_input = get_YN_input('Host Check results not found. There is no {0}. Do you want to run host checks [y/n] (y)'.format(hostcheckfileca), True)
+ if run_check_hosts_input:
+ h.run_check_hosts()
+
with open(TMP_HOST_CHECK_FILE_NAME, "wb") as tmp_f:
with open(hostcheckfile, "rb") as f1:
with open(hostcheckfileca, "rb") as f2:
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2e11c86/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
index a1c93b1..203b000 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
@@ -37,6 +37,8 @@ from resource_management.core.resources import Directory, File
from ambari_commons.constants import AMBARI_SUDO_BINARY
from resource_management.core import shell
+# WARNING. If you are adding a new host check that is used by cleanup, add it to BEFORE_CLEANUP_HOST_CHECKS
+# It is used by HostCleanup.py
CHECK_JAVA_HOME = "java_home_check"
CHECK_DB_CONNECTION = "db_connection_check"
CHECK_HOST_RESOLUTION = "host_resolution_check"
@@ -45,6 +47,8 @@ CHECK_INSTALLED_PACKAGES = "installed_packages"
CHECK_EXISTING_REPOS = "existing_repos"
CHECK_TRANSPARENT_HUGE_PAGE = "transparentHugePage"
+BEFORE_CLEANUP_HOST_CHECKS = ','.join([CHECK_LAST_AGENT_ENV, CHECK_INSTALLED_PACKAGES, CHECK_EXISTING_REPOS, CHECK_TRANSPARENT_HUGE_PAGE])
+
DB_MYSQL = "mysql"
DB_ORACLE = "oracle"
DB_POSTGRESQL = "postgres"
@@ -115,6 +119,8 @@ class CheckHost(Script):
#print "CONFIG: " + str(config)
check_execute_list = config['commandParams']['check_execute_list']
+ if check_execute_list == '*BEFORE_CLEANUP_HOST_CHECKS*':
+ check_execute_list = BEFORE_CLEANUP_HOST_CHECKS
structured_output = {}
# check each of the commands; if an unknown exception wasn't handled
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2e11c86/nohup.out
----------------------------------------------------------------------
diff --git a/nohup.out b/nohup.out
new file mode 100644
index 0000000..97275e2
--- /dev/null
+++ b/nohup.out
@@ -0,0 +1,16 @@
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
+[debug] execute contextualize
[2/3] ambari git commit: Revert "AMBARI-13227. Debian 7. Host check
does not show warning about THP enabled on hosts. (aonishuk)"
Posted by ao...@apache.org.
Revert "AMBARI-13227. Debian 7. Host check does not show warning about THP enabled on hosts. (aonishuk)"
This reverts commit b3d36fb5c7dcd498bb48c629c810a74161c139f3.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/abf1cb84
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/abf1cb84
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/abf1cb84
Branch: refs/heads/trunk
Commit: abf1cb84757d0f6132a7db59ea24a3c5a80f3f50
Parents: c538a4d
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Wed Sep 30 20:18:55 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Oct 1 14:43:57 2015 +0300
----------------------------------------------------------------------
.../src/main/python/ambari_agent/HostInfo.py | 14 ++++---------
.../test/python/ambari_agent/TestHostInfo.py | 22 --------------------
.../custom_actions/scripts/check_host.py | 13 ++++--------
.../test/python/custom_actions/TestCheckHost.py | 1 -
4 files changed, 8 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/abf1cb84/ambari-agent/src/main/python/ambari_agent/HostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostInfo.py b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
index 49f8417..3d7125a 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostInfo.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostInfo.py
@@ -157,8 +157,7 @@ class HostInfoLinux(HostInfo):
DEFAULT_SERVICE_NAME = "ntpd"
SERVICE_STATUS_CMD = "%s %s status" % (SERVICE_CMD, DEFAULT_SERVICE_NAME)
- THP_FILE_REDHAT = "/sys/kernel/mm/redhat_transparent_hugepage/enabled"
- THP_FILE_UBUNTU = "/sys/kernel/mm/transparent_hugepage/enabled"
+ THP_FILE = "/sys/kernel/mm/redhat_transparent_hugepage/enabled"
def __init__(self, config=None):
super(HostInfoLinux, self).__init__(config)
@@ -217,15 +216,10 @@ class HostInfoLinux(HostInfo):
pass
def getTransparentHugePage(self):
+ # This file exist only on redhat 6
thp_regex = "\[(.+)\]"
- file_name = None
- if OSCheck.is_ubuntu_family():
- file_name = self.THP_FILE_UBUNTU
- elif OSCheck.is_redhat_family():
- file_name = self.THP_FILE_REDHAT
-
- if file_name:
- with open(file_name) as f:
+ if os.path.isfile(self.THP_FILE):
+ with open(self.THP_FILE) as f:
file_content = f.read()
return re.search(thp_regex, file_content).groups()[0]
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/abf1cb84/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index a982f5f..57e4224 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -537,27 +537,5 @@ class TestHostInfo(TestCase):
os_path_isfile_mock.return_value = False
self.assertEqual("", hostInfo.getTransparentHugePage())
- @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = ('debian','7','Final')))
- @patch("os.path.isfile")
- @patch('__builtin__.open')
- def test_transparent_huge_page_debian(self, open_mock, os_path_isfile_mock):
- context_manager_mock = MagicMock()
- open_mock.return_value = context_manager_mock
- file_mock = MagicMock()
- file_mock.read.return_value = "[never] always"
- enter_mock = MagicMock()
- enter_mock.return_value = file_mock
- exit_mock = MagicMock()
- setattr( context_manager_mock, '__enter__', enter_mock )
- setattr( context_manager_mock, '__exit__', exit_mock )
-
- hostInfo = HostInfoLinux()
-
- os_path_isfile_mock.return_value = True
- self.assertEqual("never", hostInfo.getTransparentHugePage())
-
- os_path_isfile_mock.return_value = False
- self.assertEqual("", hostInfo.getTransparentHugePage())
-
if __name__ == "__main__":
unittest.main()
http://git-wip-us.apache.org/repos/asf/ambari/blob/abf1cb84/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
index 9aa3afb..a1c93b1 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
@@ -69,8 +69,7 @@ JARS_PATH_IN_ARCHIVE_SQLA = "/sqla-client-jdbc/java"
LIBS_PATH_IN_ARCHIVE_SQLA = "/sqla-client-jdbc/native/lib64"
JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE = "/sqla-client-jdbc/java/" + JDBC_DRIVER_SQLA_JAR
-THP_FILE_REDHAT = "/sys/kernel/mm/redhat_transparent_hugepage/enabled"
-THP_FILE_UBUNTU = "/sys/kernel/mm/transparent_hugepage/enabled"
+THP_FILE = "/sys/kernel/mm/redhat_transparent_hugepage/enabled"
class CheckHost(Script):
# Packages that are used to find repos (then repos are used to find other packages)
@@ -166,14 +165,10 @@ class CheckHost(Script):
# Here we are checking transparent huge page if CHECK_TRANSPARENT_HUGE_PAGE is in check_execute_list
if CHECK_TRANSPARENT_HUGE_PAGE in check_execute_list:
try :
+ # This file exist only on redhat 6
thp_regex = "\[(.+)\]"
- file_name = None
- if OSCheck.is_ubuntu_family():
- file_name = THP_FILE_UBUNTU
- elif OSCheck.is_redhat_family():
- file_name = THP_FILE_REDHAT
- if os.path.isfile(file_name):
- with open(file_name) as f:
+ if os.path.isfile(THP_FILE):
+ with open(THP_FILE) as f:
file_content = f.read()
structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = {"exit_code" : 0, "message": str(re.search(thp_regex,
file_content).groups()[0])}
http://git-wip-us.apache.org/repos/asf/ambari/blob/abf1cb84/ambari-server/src/test/python/custom_actions/TestCheckHost.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/custom_actions/TestCheckHost.py b/ambari-server/src/test/python/custom_actions/TestCheckHost.py
index 1e45927..b21254a 100644
--- a/ambari-server/src/test/python/custom_actions/TestCheckHost.py
+++ b/ambari-server/src/test/python/custom_actions/TestCheckHost.py
@@ -344,7 +344,6 @@ class TestCheckHost(TestCase):
pass
- @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = ('debian','7','Final')))
@patch.object(HostCheckReportFileHandler, "resolve_ambari_config")
@patch("resource_management.libraries.script.Script.put_structured_out")
@patch.object(Script, 'get_tmp_dir')
[3/3] ambari git commit: AMBARI-13284. HostCleanup script does not
work for hosts that are manually registered (aonishuk)
Posted by ao...@apache.org.
AMBARI-13284. HostCleanup script does not work for hosts that are manually registered (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/055e184c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/055e184c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/055e184c
Branch: refs/heads/branch-2.1
Commit: 055e184c91d828359a1d57c6e07df01b610b733a
Parents: 719bea4
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Oct 1 14:44:00 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Oct 1 14:44:00 2015 +0300
----------------------------------------------------------------------
.../src/main/python/ambari_agent/HostCleanup.py | 28 +++++++++++++++++++-
.../custom_actions/scripts/check_host.py | 6 +++++
2 files changed, 33 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/055e184c/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
index a725d43..4d3cd83 100644
--- a/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
+++ b/ambari-agent/src/main/python/ambari_agent/HostCleanup.py
@@ -34,7 +34,9 @@ import ConfigParser
import optparse
import shlex
import datetime
+import tempfile
from AmbariConfig import AmbariConfig
+from ambari_agent.Constants import AGENT_TMP_DIR
from ambari_commons import OSCheck, OSConst
from ambari_commons.constants import AMBARI_SUDO_BINARY
from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl
@@ -46,6 +48,7 @@ GROUP_ERASE_CMD = "groupdel {0}"
PROC_KILL_CMD = "kill -9 {0}"
ALT_DISP_CMD = "alternatives --display {0}"
ALT_ERASE_CMD = "alternatives --remove {0} {1}"
+RUN_HOST_CHECKS_CMD = '/var/lib/ambari-agent/cache/custom_actions/scripts/check_host.py ACTIONEXECUTE {0} /var/lib/ambari-agent/cache/custom_actions {1} INFO {2}'
REPO_PATH_RHEL = "/etc/yum.repos.d"
REPO_PATH_SUSE = "/etc/zypp/repos.d/"
@@ -492,6 +495,19 @@ class HostCleanup:
(stdoutdata, stderrdata) = process.communicate()
return process.returncode, stdoutdata, stderrdata
+ def run_check_hosts(self):
+ config_json = '{"commandParams": {"check_execute_list": "*BEFORE_CLEANUP_HOST_CHECKS*"}}'
+ with tempfile.NamedTemporaryFile(delete=False) as config_json_file:
+ config_json_file.write(config_json)
+
+ with tempfile.NamedTemporaryFile(delete=False) as tmp_output_file:
+ tmp_output_file.write('{}')
+
+ run_checks_command = RUN_HOST_CHECKS_CMD.format(config_json_file.name, tmp_output_file.name, AGENT_TMP_DIR)
+ (returncode, stdoutdata, stderrdata) = self.run_os_command(run_checks_command)
+ if returncode != 0:
+ logger.warn('Failed to run host checks,\nstderr:\n ' + stderrdata + '\n\nstdout:\n' + stdoutdata)
+
# Copy file and save with file.# (timestamp)
def backup_file(filePath):
if filePath is not None and os.path.exists(filePath):
@@ -584,7 +600,17 @@ def main():
sys.exit(1)
hostcheckfile, hostcheckfileca = options.inputfiles.split(",")
-
+
+ # Manage non UI install
+ if not os.path.exists(hostcheckfileca):
+ if options.silent:
+ print 'Host Check results not found. There is no {0}. Running host checks.'.format(hostcheckfileca)
+ h.run_check_hosts()
+ else:
+ run_check_hosts_input = get_YN_input('Host Check results not found. There is no {0}. Do you want to run host checks [y/n] (y)'.format(hostcheckfileca), True)
+ if run_check_hosts_input:
+ h.run_check_hosts()
+
with open(TMP_HOST_CHECK_FILE_NAME, "wb") as tmp_f:
with open(hostcheckfile, "rb") as f1:
with open(hostcheckfileca, "rb") as f2:
http://git-wip-us.apache.org/repos/asf/ambari/blob/055e184c/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
index 4d3cf76..12483c5 100644
--- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
+++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py
@@ -37,6 +37,8 @@ from resource_management.core.resources import Directory, File
from ambari_commons.constants import AMBARI_SUDO_BINARY
from resource_management.core import shell
+# WARNING. If you are adding a new host check that is used by cleanup, add it to BEFORE_CLEANUP_HOST_CHECKS
+# It is used by HostCleanup.py
CHECK_JAVA_HOME = "java_home_check"
CHECK_DB_CONNECTION = "db_connection_check"
CHECK_HOST_RESOLUTION = "host_resolution_check"
@@ -45,6 +47,8 @@ CHECK_INSTALLED_PACKAGES = "installed_packages"
CHECK_EXISTING_REPOS = "existing_repos"
CHECK_TRANSPARENT_HUGE_PAGE = "transparentHugePage"
+BEFORE_CLEANUP_HOST_CHECKS = ','.join([CHECK_LAST_AGENT_ENV, CHECK_INSTALLED_PACKAGES, CHECK_EXISTING_REPOS, CHECK_TRANSPARENT_HUGE_PAGE])
+
DB_MYSQL = "mysql"
DB_ORACLE = "oracle"
DB_POSTGRESQL = "postgres"
@@ -115,6 +119,8 @@ class CheckHost(Script):
#print "CONFIG: " + str(config)
check_execute_list = config['commandParams']['check_execute_list']
+ if check_execute_list == '*BEFORE_CLEANUP_HOST_CHECKS*':
+ check_execute_list = BEFORE_CLEANUP_HOST_CHECKS
structured_output = {}
# check each of the commands; if an unknown exception wasn't handled