You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2017/10/11 23:03:32 UTC
[05/50] [abbrv] ambari git commit: AMBARI-22139 - CURRENT cluster
Shows Upgrade If Component Didn't Report Version (jonathanhurley)
AMBARI-22139 - CURRENT cluster Shows Upgrade If Component Didn't Report Version (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8b83a0a5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8b83a0a5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8b83a0a5
Branch: refs/heads/branch-feature-AMBARI-14714-ui
Commit: 8b83a0a538358e54c5aa7f3c4eccc4a458296f9a
Parents: a9d2698
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu Oct 5 16:06:06 2017 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Oct 6 10:38:29 2017 -0400
----------------------------------------------------------------------
.../python/resource_management/TestScript.py | 26 ++++++++-
.../libraries/script/script.py | 22 ++++++--
.../0.4.0/package/scripts/ranger_admin.py | 13 ++---
.../configs/ranger_admin_default.json | 55 ++++++++++++++++++++
.../src/test/python/stacks/utils/RMFTestCase.py | 9 ++--
5 files changed, 110 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-agent/src/test/python/resource_management/TestScript.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestScript.py b/ambari-agent/src/test/python/resource_management/TestScript.py
index 75726d6..79d0598 100644
--- a/ambari-agent/src/test/python/resource_management/TestScript.py
+++ b/ambari-agent/src/test/python/resource_management/TestScript.py
@@ -21,9 +21,9 @@ import StringIO
import sys, pprint
from resource_management.libraries.script import Script
from resource_management.core.environment import Environment
+from resource_management.core.logger import Logger
from mock.mock import patch, MagicMock
from stacks.utils.RMFTestCase import *
-import logging
class TestScript(RMFTestCase):
@@ -110,7 +110,7 @@ class TestScript(RMFTestCase):
@patch("__builtin__.open")
def test_status_commands_clear_structured_out(self, open_mock):
"""
- Tests that status commands will clear and stored structured output from prior status commands.
+ Tests that status commands will clear any stored structured output from prior status commands.
:param open_mock:
:return:
"""
@@ -141,6 +141,28 @@ class TestScript(RMFTestCase):
self.assertTrue(open_mock.called)
self.assertEquals({}, Script.structuredOut)
+
+ @patch.object(Logger, "error", new = MagicMock())
+ @patch.object(Script, "put_structured_out")
+ @patch("resource_management.libraries.functions.version_select_util.get_component_version_from_symlink", new = MagicMock(return_value=None))
+ @patch("resource_management.libraries.functions.stack_select.get_package_name", new = MagicMock(return_value="foo-package"))
+ @patch("resource_management.libraries.functions.stack_select.unsafe_get_stack_versions", new = MagicMock(return_value=("",0,["2.6.0.0-1234"])))
+ def test_save_version_structured_out_stack_select(self, pso_mock):
+ """
+ Tests that when writing out the version of the component to the structure output,
+ if all else fails, we'll invoke the stack-select tool to see if there are any versions
+ reported.
+ :param pso_mock:
+ :return:
+ """
+ script = Script()
+ script.stroutfile = ''
+ script.save_component_version_to_structured_out("start")
+
+ self.assertEqual(pso_mock.call_count, 1)
+ self.assertEquals(pso_mock.call_args[0][0], {'version':'2.6.0.0-1234'})
+
+
def tearDown(self):
# enable stdout
sys.stdout = sys.__stdout__
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index e612638..d5b4469 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -47,7 +47,7 @@ from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning
from resource_management.core.resources.packaging import Package
-from resource_management.libraries.functions.version_select_util import get_component_version_from_symlink
+from resource_management.libraries.functions import version_select_util
from resource_management.libraries.functions.version import compare_versions
from resource_management.libraries.functions.version import format_stack_version
from resource_management.libraries.functions import stack_tools
@@ -212,6 +212,12 @@ class Script(object):
Saves the version of the component for this command to the structured out file. If the
command is an install command and the repository is trusted, then it will use the version of
the repository. Otherwise, it will consult the stack-select tool to read the symlink version.
+
+ Under rare circumstances, a component may have a bug which prevents it from reporting a
+ version back after being installed. This is most likely due to the stack-select tool not being
+ invoked by the package's installer. In these rare cases, we try to see if the component
+ should have reported a version and we try to fallback to the "<stack-select> versions" command.
+
:param command_name: command name
:return: None
"""
@@ -240,7 +246,17 @@ class Script(object):
if stack_select_package_name and stack_name:
# only query for the component version from stack-select if we can't trust the repository yet
if component_version is None:
- component_version = get_component_version_from_symlink(stack_name, stack_select_package_name)
+ component_version = version_select_util.get_component_version_from_symlink(stack_name, stack_select_package_name)
+
+ # last ditch effort - should cover the edge case where the package failed to setup its
+ # link and we have to try to see if <stack-select> can help
+ if component_version is None:
+ output, code, versions = stack_select.unsafe_get_stack_versions()
+ if len(versions) == 1:
+ component_version = versions[0]
+ Logger.error("The '{0}' component did not advertise a version. This may indicate a problem with the component packaging. " \
+ "However, the stack-select tool was able to report a single version installed ({1}). " \
+ "This is the version that will be reported.".format(stack_select_package_name, component_version))
if component_version:
self.put_structured_out({"version": component_version})
@@ -252,7 +268,7 @@ class Script(object):
self.put_structured_out({"repository_version_id": repo_version_id})
else:
if not self.is_hook():
- Logger.error("Component '{0}' did not advertise a version. This may indicate a problem with the component packaging.".format(stack_select_package_name))
+ Logger.error("The '{0}' component did not advertise a version. This may indicate a problem with the component packaging.".format(stack_select_package_name))
def should_expose_component_version(self, command_name):
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
index f779c18..848b137 100644
--- a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
+++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/package/scripts/ranger_admin.py
@@ -20,7 +20,6 @@ limitations under the License.
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.check_process_status import check_process_status
from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions.constants import Direction
from resource_management.libraries.script import Script
from resource_management.core.resources.system import Execute, File
from resource_management.core.exceptions import ComponentIsNotRunning
@@ -28,10 +27,12 @@ from resource_management.libraries.functions.format import format
from resource_management.core.logger import Logger
from resource_management.core import shell
from ranger_service import ranger_service
-from setup_ranger_xml import setup_ranger_audit_solr, setup_ranger_admin_passwd_change, update_password_configs
from resource_management.libraries.functions import solr_cloud_util
-from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING, UPGRADE_TYPE_ROLLING
+from ambari_commons.constants import UPGRADE_TYPE_NON_ROLLING
from resource_management.libraries.functions.constants import Direction
+
+import setup_ranger_xml
+
import os, errno
class RangerAdmin(Script):
@@ -93,9 +94,9 @@ class RangerAdmin(Script):
if params.stack_supports_infra_client and params.audit_solr_enabled and params.is_solrCloud_enabled:
solr_cloud_util.setup_solr_client(params.config, custom_log4j = params.custom_log4j)
- setup_ranger_audit_solr()
+ setup_ranger_xml.setup_ranger_audit_solr()
- update_password_configs()
+ setup_ranger_xml.update_password_configs()
ranger_service('ranger_admin')
@@ -142,7 +143,7 @@ class RangerAdmin(Script):
setup_java_patch()
if params.stack_supports_ranger_admin_password_change:
- setup_ranger_admin_passwd_change()
+ setup_ranger_xml.setup_ranger_admin_passwd_change()
def set_ru_rangeradmin_in_progress(self, upgrade_marker_file):
config_dir = os.path.dirname(upgrade_marker_file)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json b/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json
index b360c07..2e66c62 100644
--- a/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json
+++ b/ambari-server/src/test/python/common-services/configs/ranger_admin_default.json
@@ -64,6 +64,61 @@
"db_host": "localhost",
"xa_ldap_groupRoleAttribute": "\"cn\""
},
+ "ranger-ugsync-site": {
+ "ranger.usersync.ldap.binddn": "",
+ "ranger.usersync.policymgr.username": "rangerusersync",
+ "ranger.usersync.policymanager.mockrun": "false",
+ "ranger.usersync.group.searchbase": "",
+ "ranger.usersync.ldap.bindalias": "testldapalias",
+ "ranger.usersync.truststore.file": "/usr/hdp/current/ranger-usersync/conf/mytruststore.jks",
+ "ranger.usersync.port": "5151",
+ "ranger.usersync.pagedresultssize": "500",
+ "ranger.usersync.group.memberattributename": "",
+ "ranger.usersync.kerberos.principal": "",
+ "ranger.usersync.source.impl.class": "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder",
+ "ranger.usersync.ldap.referral": "ignore",
+ "ranger.usersync.group.searchfilter": "",
+ "ranger.usersync.ldap.user.objectclass": "person",
+ "ranger.usersync.logdir": "{{usersync_log_dir}}",
+ "ranger.usersync.ldap.user.searchfilter": "",
+ "ranger.usersync.ldap.groupname.caseconversion": "none",
+ "ranger.usersync.ldap.ldapbindpassword": "",
+ "ranger.usersync.unix.minUserId": "500",
+ "ranger.usersync.policymanager.maxrecordsperapicall": "1000",
+ "ranger.usersync.group.nameattribute": "",
+ "ranger.usersync.policymgr.alias": "ranger.usersync.policymgr.password",
+ "ranger.usersync.keystore.file": "/usr/hdp/current/ranger-usersync/conf/unixauthservice.jks",
+ "ranger.usersync.user.searchenabled": "false",
+ "ranger.usersync.group.usermapsyncenabled": "true",
+ "ranger.usersync.ldap.bindkeystore": "",
+ "ranger.usersync.ldap.user.groupnameattribute": "memberof, ismemberof",
+ "ranger.usersync.kerberos.keytab": "",
+ "ranger.usersync.passwordvalidator.path": "./native/credValidator.uexe",
+ "ranger.usersync.group.objectclass": "",
+ "ranger.usersync.ldap.user.searchscope": "sub",
+ "ranger.usersync.unix.password.file": "/etc/passwd",
+ "ranger.usersync.ldap.user.nameattribute": "",
+ "ranger.usersync.pagedresultsenabled": "true",
+ "ranger.usersync.policymanager.baseURL": "{{ranger_external_url}}",
+ "ranger.usersync.group.search.first.enabled": "false",
+ "ranger.usersync.group.searchenabled": "false",
+ "ranger.usersync.sink.impl.class": "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder",
+ "ranger.usersync.ssl": "true",
+ "ranger.usersync.ldap.url": "",
+ "ranger.usersync.ldap.searchBase": "dc=hadoop,dc=apache,dc=org",
+ "ranger.usersync.policymgr.keystore": "/usr/hdp/current/ranger-usersync/conf/ugsync.jceks",
+ "ranger.usersync.ldap.user.searchbase": "",
+ "ranger.usersync.ldap.username.caseconversion": "none",
+ "ranger.usersync.credstore.filename": "/usr/hdp/current/ranger-usersync/conf/ugsync.jceks",
+ "ranger.usersync.keystore.password": "UnIx529p",
+ "ranger.usersync.unix.group.file": "/etc/group",
+ "ranger.usersync.filesource.file": "/tmp/usergroup.txt",
+ "ranger.usersync.group.searchscope": "",
+ "ranger.usersync.truststore.password": "changeit",
+ "ranger.usersync.enabled": "true",
+ "ranger.usersync.sleeptimeinmillisbetweensynccycle": "60000",
+ "ranger.usersync.filesource.text.delimiter": ","
+ },
"ranger-site": {
"http.enabled": "true",
"http.service.port": "6080",
http://git-wip-us.apache.org/repos/asf/ambari/blob/8b83a0a5/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
index d98e0b1..bff8642 100644
--- a/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
+++ b/ambari-server/src/test/python/stacks/utils/RMFTestCase.py
@@ -154,11 +154,12 @@ class RMFTestCase(TestCase):
with patch('resource_management.libraries.functions.stack_select.is_package_supported', return_value=True):
with patch('resource_management.libraries.functions.stack_select.get_supported_packages', return_value=MagicMock()):
with patch.object(os, "environ", new=os_env) as mocks_dict['environ']:
- if not try_install:
- with patch.object(Script, 'install_packages') as install_mock_value:
+ with patch('resource_management.libraries.functions.stack_select.unsafe_get_stack_versions', return_value = (("",0,[]))):
+ if not try_install:
+ with patch.object(Script, 'install_packages') as install_mock_value:
+ method(RMFTestCase.env, *command_args)
+ else:
method(RMFTestCase.env, *command_args)
- else:
- method(RMFTestCase.env, *command_args)
sys.path.remove(scriptsdir)