You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/01/19 19:26:56 UTC
[01/19] ambari git commit: AMBARI-14709. RU: HBase service check
failed during upgrade 2.2.8->2.4 (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/branch-dev-patch-upgrade f40df6b57 -> 4b36397ff
AMBARI-14709. RU: HBase service check failed during upgrade 2.2.8->2.4 (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/40aa6121
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/40aa6121
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/40aa6121
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 40aa6121fa9cc0a3a39f02b2fb7acbe612f426ad
Parents: 5164bc5
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Jan 18 17:02:29 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Jan 18 17:02:29 2016 +0200
----------------------------------------------------------------------
.../src/main/python/resource_management/core/shell.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/40aa6121/ambari-common/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/shell.py b/ambari-common/src/main/python/resource_management/core/shell.py
index 5acbda9..fac264c 100644
--- a/ambari-common/src/main/python/resource_management/core/shell.py
+++ b/ambari-common/src/main/python/resource_management/core/shell.py
@@ -141,7 +141,7 @@ def _call_wrapper(command, **kwargs):
break
except ExecuteTimeoutException as ex:
if on_timeout:
- Logger.info("Executing '%s'. Reason: %s" % (on_timeout, str(ex)))
+ Logger.info("Executing '{0}'. Reason: {1}".format(on_timeout, str(ex)))
result = checked_call(on_timeout)
else:
raise
@@ -149,7 +149,7 @@ def _call_wrapper(command, **kwargs):
if is_last_try: # last try
raise
else:
- Logger.info("Retrying after %d seconds. Reason: %s" % (try_sleep, str(ex)))
+ Logger.info("Retrying after {0} seconds. Reason: {1}".format(try_sleep, str(ex)))
time.sleep(try_sleep)
return result
@@ -283,13 +283,13 @@ def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE
t.cancel()
# timeout occurred
else:
- err_msg = Logger.filter_text(("Execution of '%s' was killed due timeout after %d seconds") % (command_alias, timeout))
+ err_msg = "Execution of '{0}' was killed due timeout after {1} seconds".format(command, timeout)
raise ExecuteTimeoutException(err_msg)
code = proc.returncode
if throw_on_failure and code:
- err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_alias, code, all_output))
+ err_msg = Logger.filter_text("Execution of '{0}' returned {1}. {2}".format(command_alias, code, all_output))
raise Fail(err_msg)
# if separate stderr is enabled (by default it's redirected to out)
@@ -313,7 +313,7 @@ def as_sudo(command, env=None, auto_escape=True):
#
# In that case while passing string,
# any bash symbols eventually added to command like && || ; < > | << >> would cause problems.
- err_msg = Logger.filter_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple of arguments") % (command))
+ err_msg = Logger.filter_text("String command '{0}' cannot be run as sudo. Please supply the command as a tuple of arguments".format(command))
raise Fail(err_msg)
env = _get_environment_str(_add_current_path_to_env(env)) if env else ENV_PLACEHOLDER
[13/19] ambari git commit: AMBARI-14716 Prevent the user from
removing the last service. (atkach)
Posted by nc...@apache.org.
AMBARI-14716 Prevent the user from removing the last service. (atkach)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/39400f4e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/39400f4e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/39400f4e
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 39400f4e97233a9d19391a28a0e47008a270d316
Parents: 869e085
Author: Andrii Tkach <at...@hortonworks.com>
Authored: Tue Jan 19 11:37:45 2016 +0200
Committer: Andrii Tkach <at...@hortonworks.com>
Committed: Tue Jan 19 11:37:45 2016 +0200
----------------------------------------------------------------------
ambari-web/app/controllers/main/service/item.js | 25 +++++++++++++-------
ambari-web/app/messages.js | 3 ++-
.../test/controllers/main/service/item_test.js | 12 ++++++++++
3 files changed, 31 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/39400f4e/ambari-web/app/controllers/main/service/item.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/item.js b/ambari-web/app/controllers/main/service/item.js
index 260b1d2..2c1d4cc 100644
--- a/ambari-web/app/controllers/main/service/item.js
+++ b/ambari-web/app/controllers/main/service/item.js
@@ -954,26 +954,35 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow
*/
deleteService: function(serviceName) {
var dependentServices = App.StackService.find(serviceName).get('requiredServices').filter(function(_serviceName) {
- return App.Service.find(_serviceName).get('isLoaded');
- });
- var self = this;
- var displayName = App.format.role(serviceName);
-
- if (dependentServices.length > 0) {
+ return App.Service.find(_serviceName).get('isLoaded');
+ }),
+ self = this,
+ displayName = App.format.role(serviceName),
+ popupHeader = Em.I18n.t('services.service.delete.popup.header');
+
+ if (App.Service.find().get('length') === 1) {
+ //at least one service should be installed
+ App.ModalPopup.show({
+ secondary: null,
+ header: popupHeader,
+ encodeBody: false,
+ body: Em.I18n.t('services.service.delete.lastService.popup.body').format(displayName)
+ });
+ } else if (dependentServices.length > 0) {
this.dependentServicesWarning(serviceName, dependentServices);
} else if (App.Service.find(serviceName).get('workStatus') === 'INSTALLED') {
App.showConfirmationPopup(
function() {self.confirmDeleteService(serviceName)},
Em.I18n.t('services.service.delete.popup.warning').format(displayName),
null,
- Em.I18n.t('services.service.delete.popup.header'),
+ popupHeader,
Em.I18n.t('common.delete'),
true
);
} else {
App.ModalPopup.show({
secondary: null,
- header: Em.I18n.t('services.service.delete.popup.header'),
+ header: popupHeader,
encodeBody: false,
body: Em.I18n.t('services.service.delete.popup.mustBeStopped').format(displayName)
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/39400f4e/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index 203e7ee..82797a7 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -1679,10 +1679,11 @@ Em.I18n.translations = {
'services.service.actions.serviceActions':'Service Actions',
'services.service.delete.popup.header': 'Delete Service',
+ 'services.service.delete.lastService.popup.body': 'The <b>{0}</b> service can\'t be deleted, at least one service must be installed.',
'services.service.delete.popup.dependentServices': 'Prior to deleting <b>{0}</b>, you must delete the following dependent services:',
'services.service.delete.popup.mustBeStopped': 'Prior to deleting <b>{0}</b>, you must stop the service.',
'services.service.delete.popup.warning': 'The <b>{0} service will be removed from Ambari and all configurations' +
- ' and configuration history will be lost</b>',
+ ' and configuration history will be lost.</b>',
'services.service.confirmDelete.popup.header': 'Confirm Delete',
'services.service.confirmDelete.popup.body': 'You must confirm delete of <b>{0}</b> by typing "yes"' +
' in the confirmation box. <b>This operation is not reversible and all configuration history will be lost.</b>',
http://git-wip-us.apache.org/repos/asf/ambari/blob/39400f4e/ambari-web/test/controllers/main/service/item_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/main/service/item_test.js b/ambari-web/test/controllers/main/service/item_test.js
index 98ff890..55d27f7 100644
--- a/ambari-web/test/controllers/main/service/item_test.js
+++ b/ambari-web/test/controllers/main/service/item_test.js
@@ -1197,6 +1197,18 @@ describe('App.MainServiceItemController', function () {
App.format.role.restore();
});
+ it("only one service installed", function() {
+ this.mockStackService.returns(Em.Object.create({requiredServices: ['S2']}));
+ this.mockService.returns(Em.Object.create({length: 1}));
+ mainServiceItemController.deleteService('S1');
+ expect(App.ModalPopup.show.calledWith({
+ secondary: null,
+ header: Em.I18n.t('services.service.delete.popup.header'),
+ encodeBody: false,
+ body: Em.I18n.t('services.service.delete.lastService.popup.body').format('S1')
+ })).to.be.true;
+ });
+
it("service has installed dependent services", function() {
this.mockStackService.returns(Em.Object.create({requiredServices: ['S2']}));
this.mockService.returns(Em.Object.create({workStatus: 'INSTALLED', isLoaded: true}));
[15/19] ambari git commit: AMBARI-12430. Improve checks for
ambari.properties file for missing/needed properties during startup
(aonishuk)
Posted by nc...@apache.org.
AMBARI-12430. Improve checks for ambari.properties file for missing/needed properties during startup (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/48246c9f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/48246c9f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/48246c9f
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 48246c9fbdbe13b73e9851df58c1813fc134f635
Parents: 1c2c6a9
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Jan 19 15:48:11 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Jan 19 15:48:11 2016 +0200
----------------------------------------------------------------------
.../python/ambari_server/serverConfiguration.py | 15 ++++++--
.../src/test/python/TestAmbariServer.py | 38 ++++++++++++++++++--
2 files changed, 48 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/48246c9f/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index aeb4d5b..d5b9e86 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -177,10 +177,20 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s
DEFAULT_DB_NAME = "ambari"
+SECURITY_KEYS_DIR = "security.server.keys_dir"
COMMON_SERVICES_PATH_PROPERTY = 'common.services.path'
WEBAPP_DIR_PROPERTY = 'webapp.dir'
-REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY, SERVER_VERSION_FILE_PATH, \
- WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY]
+SHARED_RESOURCES_DIR = 'shared.resources.dir'
+BOOTSTRAP_SCRIPT = 'bootstrap.script'
+CUSTOM_ACTION_DEFINITIONS = 'custom.action.definitions'
+BOOTSTRAP_SETUP_AGENT_SCRIPT = 'bootstrap.setup_agent.script'
+STACKADVISOR_SCRIPT = 'stackadvisor.script'
+REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY, SERVER_VERSION_FILE_PATH,
+ WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY, SECURITY_KEYS_DIR, JDBC_DATABASE_NAME_PROPERTY,
+ NR_USER_PROPERTY, JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY,
+ JDBC_PASSWORD_PROPERTY, SHARED_RESOURCES_DIR, JDBC_USER_NAME_PROPERTY, BOOTSTRAP_SCRIPT,
+ RESOURCES_DIR_PROPERTY, CUSTOM_ACTION_DEFINITIONS, BOOTSTRAP_SETUP_AGENT_SCRIPT,
+ STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY]
class ServerDatabaseType(object):
internal = 0
@@ -457,7 +467,6 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
configDefaults = ServerConfigDefaults()
# Security
-SECURITY_KEYS_DIR = "security.server.keys_dir"
SECURITY_MASTER_KEY_LOCATION = "security.master.key.location"
SECURITY_KEY_IS_PERSISTED = "security.master.key.ispersisted"
SECURITY_KEY_ENV_VAR_NAME = "AMBARI_SECURITY_MASTER_KEY"
http://git-wip-us.apache.org/repos/asf/ambari/blob/48246c9f/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 970bcab..ad4505a 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -76,7 +76,9 @@ with patch("platform.linux_distribution", return_value = os_distro_value):
SSL_TRUSTSTORE_PASSWORD_PROPERTY, SECURITY_IS_ENCRYPTION_ENABLED, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
SECURITY_MASTER_KEY_LOCATION, SECURITY_KEYS_DIR, LDAP_PRIMARY_URL_PROPERTY, store_password_file, \
get_pass_file_path, GET_FQDN_SERVICE_URL, JDBC_USE_INTEGRATED_AUTH_PROPERTY, SECURITY_KEY_ENV_VAR_NAME, \
- JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY
+ JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, STACK_LOCATION_KEY, SERVER_VERSION_FILE_PATH, \
+ COMMON_SERVICES_PATH_PROPERTY, WEBAPP_DIR_PROPERTY, SHARED_RESOURCES_DIR, BOOTSTRAP_SCRIPT, \
+ CUSTOM_ACTION_DEFINITIONS, BOOTSTRAP_SETUP_AGENT_SCRIPT, STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY
from ambari_server.serverUtils import is_server_runing, refresh_stack_hash
from ambari_server.serverSetup import check_selinux, check_ambari_user, proceedJDBCProperties, SE_STATUS_DISABLED, SE_MODE_ENFORCING, configure_os_settings, \
download_and_install_jdk, prompt_db_properties, setup, \
@@ -4348,6 +4350,7 @@ class TestAmbariServer(TestCase):
del args.jdbc_url
del args.debug
del args.suspend_start
+ args.skip_properties_validation = False
return args
@@ -4368,6 +4371,27 @@ class TestAmbariServer(TestCase):
p = Properties()
p.process_pair(SECURITY_IS_ENCRYPTION_ENABLED, 'False')
+ p.process_pair(JDBC_DATABASE_NAME_PROPERTY, 'some_value')
+ p.process_pair(NR_USER_PROPERTY, 'some_value')
+ p.process_pair(STACK_LOCATION_KEY, 'some_value')
+ p.process_pair(SERVER_VERSION_FILE_PATH, 'some_value')
+ p.process_pair(OS_TYPE_PROPERTY, 'some_value')
+ p.process_pair(JAVA_HOME_PROPERTY, 'some_value')
+ p.process_pair(JDK_NAME_PROPERTY, 'some_value')
+ p.process_pair(JCE_NAME_PROPERTY, 'some_value')
+ p.process_pair(COMMON_SERVICES_PATH_PROPERTY, 'some_value')
+ p.process_pair(JDBC_PASSWORD_PROPERTY, 'some_value')
+ p.process_pair(WEBAPP_DIR_PROPERTY, 'some_value')
+ p.process_pair(SHARED_RESOURCES_DIR, 'some_value')
+ p.process_pair(SECURITY_KEYS_DIR, 'some_value')
+ p.process_pair(JDBC_USER_NAME_PROPERTY, 'some_value')
+ p.process_pair(BOOTSTRAP_SCRIPT, 'some_value')
+ p.process_pair(OS_FAMILY_PROPERTY, 'some_value')
+ p.process_pair(RESOURCES_DIR_PROPERTY, 'some_value')
+ p.process_pair(CUSTOM_ACTION_DEFINITIONS, 'some_value')
+ p.process_pair(BOOTSTRAP_SETUP_AGENT_SCRIPT, 'some_value')
+ p.process_pair(STACKADVISOR_SCRIPT, 'some_value')
+ p.process_pair(BOOTSTRAP_DIR_PROPERTY, 'some_value')
get_ambari_properties_5_mock.return_value = get_ambari_properties_4_mock.return_value = \
get_ambari_properties_3_mock.return_value = get_ambari_properties_2_mock.return_value = \
@@ -4682,7 +4706,17 @@ class TestAmbariServer(TestCase):
self.assertTrue(save_master_key_method.called)
popen_arg = popenMock.call_args[1]['env']
self.assertEquals(os_environ_mock.copy.return_value, popen_arg)
- pass
+
+ # Checking situation when required properties not set up
+ args = reset_mocks()
+ p.removeProp(JAVA_HOME_PROPERTY)
+ get_ambari_properties_mock.return_value = p
+ try:
+ _ambari_server_.start(args)
+ self.fail("Should fail with 'Required properties are not found:'")
+ except FatalException as e:
+ # Expected
+ self.assertTrue('Required properties are not found:' in e.reason)
@not_for_platform(PLATFORM_WINDOWS)
[06/19] ambari git commit: AMBARI-14695: HAWQSTANDBY fails to start
when HAWQ Service is started (mithmatt via jaoki)
Posted by nc...@apache.org.
AMBARI-14695: HAWQSTANDBY fails to start when HAWQ Service is started (mithmatt via jaoki)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/342363ff
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/342363ff
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/342363ff
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 342363ff0d2192eea3e87949adad1c725a4b871e
Parents: 0cbefb3
Author: Jun Aoki <ja...@apache.org>
Authored: Mon Jan 18 11:06:24 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Mon Jan 18 11:06:24 2016 -0800
----------------------------------------------------------------------
.../src/main/resources/stacks/HDP/2.3/role_command_order.json | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/342363ff/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
index 2c44387..1b6eefc 100755
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
@@ -10,8 +10,8 @@
"PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"],
"ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"],
"SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"],
- "HAWQSTANDBY-START" : ["NAMENODE-START", "DATANODE-START"],
- "HAWQMASTER-START" : ["NAMENODE-START", "DATANODE-START", "HAWQSTANDBY-START"],
+ "HAWQMASTER-START" : ["NAMENODE-START", "DATANODE-START"],
+ "HAWQSTANDBY-START" : ["NAMENODE-START", "DATANODE-START", "HAWQMASTER-START"],
"HAWQSEGMENT-START" : ["HAWQMASTER-START", "HAWQSTANDBY-START"],
"HAWQ_SERVICE_CHECK-SERVICE_CHECK" : ["HAWQMASTER-START", "HDFS_SERVICE_CHECK-SERVICE_CHECK"],
"KNOX_GATEWAY-START" : ["RANGER_USERSYNC-START", "NAMENODE-START"],
[14/19] ambari git commit: AMBARI-14718. on region server restart
ambari is setting /tmp to hbase:hadoop (aonishuk)
Posted by nc...@apache.org.
AMBARI-14718. on region server restart ambari is setting /tmp to hbase:hadoop (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1c2c6a9f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1c2c6a9f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1c2c6a9f
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 1c2c6a9f6678931936585761a242651b8a6871ce
Parents: 39400f4
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Jan 19 14:15:03 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Jan 19 14:15:03 2016 +0200
----------------------------------------------------------------------
.../HBASE/0.96.0.2.0/configuration/hbase-env.xml | 2 +-
.../HBASE/0.96.0.2.0/package/scripts/hbase.py | 5 ++---
.../HBASE/0.96.0.2.0/package/scripts/params_linux.py | 2 --
.../src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py | 4 ----
.../src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py | 6 ------
.../python/stacks/2.0.6/HBASE/test_hbase_regionserver.py | 8 --------
.../python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py | 4 ----
7 files changed, 3 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
index cabd4fd..62c7c93 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
@@ -115,7 +115,7 @@ and the -Xmn ratio (hbase_regionserver_xmn_ratio) exceeds this value.
<property>
<name>hbase_java_io_tmpdir</name>
- <value>/tmp</value>
+ <value>{{exec_tmp_dir}}/hbase_java_io_tmpdir</value>
<description>Used in hbase-env.sh as HBASE_OPTS=-Djava.io.tmpdir=java_io_tmpdir</description>
<value-attributes>
<type>directory</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
index c407a93..71ce02b 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
@@ -22,6 +22,7 @@ from resource_management import *
import sys
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
+from resource_management.core.source import InlineTemplate
@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
@@ -56,9 +57,7 @@ def hbase(name=None):
create_parents = True
)
- Directory(params.java_io_tmpdir,
- owner = params.hbase_user,
- group = params.user_group,
+ Directory(InlineTemplate(params.java_io_tmpdir).get_content(),
create_parents = True,
mode=0777
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index 7679d90..4a44676 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -36,8 +36,6 @@ from resource_management.libraries.functions import is_empty
from resource_management.libraries.functions import get_unique_id_and_date
from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.substitute_vars import substitute_vars
-
# server configurations
config = Script.get_config()
exec_tmp_dir = Script.get_tmp_dir()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
index 9f3e19e..6dc4b0b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
@@ -47,8 +47,6 @@ class TestHBaseClient(RMFTestCase):
create_parents = True,
)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
@@ -145,8 +143,6 @@ class TestHBaseClient(RMFTestCase):
create_parents = True,
)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
index 7c6d10e..13b2e33 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
@@ -243,8 +243,6 @@ class TestHBaseMaster(RMFTestCase):
create_parents = True,
)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
@@ -380,8 +378,6 @@ class TestHBaseMaster(RMFTestCase):
create_parents = True,
)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
@@ -526,8 +522,6 @@ class TestHBaseMaster(RMFTestCase):
group = 'hadoop',
create_parents = True)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
index c36bd89..58ca7ef 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
@@ -136,8 +136,6 @@ class TestHbaseRegionServer(RMFTestCase):
create_parents = True,
)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
@@ -234,8 +232,6 @@ class TestHbaseRegionServer(RMFTestCase):
create_parents = True,
)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
@@ -342,8 +338,6 @@ class TestHbaseRegionServer(RMFTestCase):
group = 'hadoop',
create_parents = True)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
@@ -458,8 +452,6 @@ class TestHbaseRegionServer(RMFTestCase):
group = 'hadoop',
create_parents = True)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
index 6f3199e..7722af0 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
@@ -238,8 +238,6 @@ class TestPhoenixQueryServer(RMFTestCase):
create_parents = True,
)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
@@ -351,8 +349,6 @@ class TestPhoenixQueryServer(RMFTestCase):
create_parents = True,
)
self.assertResourceCalled('Directory', '/tmp',
- owner = 'hbase',
- group = 'hadoop',
create_parents = True,
mode = 0777
)
[07/19] ambari git commit: AMBARI-14653. HBase Links Metric - more
links pointing to the wrong hbase.master.info.port. (Zhe (Joe) Wang via
Jaimin)
Posted by nc...@apache.org.
AMBARI-14653. HBase Links Metric - more links pointing to the wrong hbase.master.info.port. (Zhe (Joe) Wang via Jaimin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/21b6b316
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/21b6b316
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/21b6b316
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 21b6b31696718f5190a5b2cd52c469b097e00ffa
Parents: 342363f
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Mon Jan 18 11:20:58 2016 -0800
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Mon Jan 18 11:20:58 2016 -0800
----------------------------------------------------------------------
.../app/views/common/quick_view_link_view.js | 5 +-
.../test/views/common/quick_link_view_test.js | 51 ++++++++++++++++++++
2 files changed, 53 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/21b6b316/ambari-web/app/views/common/quick_view_link_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/quick_view_link_view.js b/ambari-web/app/views/common/quick_view_link_view.js
index e58e47f..604d24d 100644
--- a/ambari-web/app/views/common/quick_view_link_view.js
+++ b/ambari-web/app/views/common/quick_view_link_view.js
@@ -163,7 +163,7 @@ App.QuickViewLinks = Em.View.extend({
var sites = ['core-site', 'hdfs-site'];
if(checks){
checks.forEach(function(check){
- var protocolConfigSiteProp = Em.get(check, 'site')
+ var protocolConfigSiteProp = Em.get(check, 'site');
if (sites.indexOf(protocolConfigSiteProp) < 0){
sites.push(protocolConfigSiteProp);
}
@@ -181,8 +181,7 @@ App.QuickViewLinks = Em.View.extend({
}
}
}, this);
-
- this.set('requiredSiteNames', sites);
+ this.set('requiredSiteNames', this.get('requiredSiteNames').pushObjects(sites).uniq());
this.setQuickLinks();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/21b6b316/ambari-web/test/views/common/quick_link_view_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/common/quick_link_view_test.js b/ambari-web/test/views/common/quick_link_view_test.js
index cae4715..b9aecee 100644
--- a/ambari-web/test/views/common/quick_link_view_test.js
+++ b/ambari-web/test/views/common/quick_link_view_test.js
@@ -180,6 +180,57 @@ describe('App.QuickViewLinks', function () {
});
});
+ describe("#loadQuickLinksConfigSuccessCallback()", function () {
+ var mock;
+
+ beforeEach(function () {
+ sinon.stub(App.store, 'commit', Em.K);
+ mock = sinon.stub(quickViewLinks, 'getQuickLinksConfiguration');
+ });
+ afterEach(function () {
+ App.store.commit.restore();
+ mock.restore();
+ });
+ it("requiredSites consistent", function () {
+ var quickLinksConfigHBASE = {
+ protocol: {
+ type: "http"
+ },
+ links: [
+ {
+ port: {
+ site: "hbase-site"
+ }
+ }
+ ]
+ };
+ var quickLinksConfigYARN = {
+ protocol: {
+ checks: [
+ {
+ site: "yarn-site"
+ }
+ ],
+ type: "https"
+ },
+ links: [
+ {
+ port: {
+ site: "yarn-site"
+ }
+ }
+ ]
+ };
+ quickViewLinks.set('content.serviceName', 'HBASE');
+ mock.returns(quickLinksConfigHBASE);
+ quickViewLinks.loadQuickLinksConfigSuccessCallback({items: []});
+ quickViewLinks.set('content.serviceName', 'YARN');
+ mock.returns(quickLinksConfigYARN);
+ quickViewLinks.loadQuickLinksConfigSuccessCallback({items: []});
+ expect(quickViewLinks.get('requiredSiteNames')).to.be.eql(["core-site", "hdfs-site", "hbase-site", "yarn-site"]);
+ });
+ });
+
describe("#getQuickLinksHosts()", function () {
beforeEach(function () {
sinon.stub(App.ajax, 'send');
[16/19] ambari git commit: AMBARI-14721. yarn-site properties for
Spark Shuffle Aux should be added unconditionally (aonishuk)
Posted by nc...@apache.org.
AMBARI-14721. yarn-site properties for Spark Shuffle Aux should be added unconditionally (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8f6b45d5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8f6b45d5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8f6b45d5
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8f6b45d5a13179cc49d0d4a857c5c157d757bb11
Parents: 48246c9
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Jan 19 17:03:11 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Jan 19 17:03:11 2016 +0200
----------------------------------------------------------------------
.../upgrades/SparkShufflePropertyConfig.java | 14 ++--
.../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 2 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 2 +-
.../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 2 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 2 +-
.../services/YARN/configuration/yarn-site.xml | 14 +---
.../stacks/HDP/2.4/services/stack_advisor.py | 78 +-------------------
.../SparkShufflePropertyConfigTest.java | 43 +----------
8 files changed, 12 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
index efeb4a0..299a373 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
@@ -68,9 +68,6 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
return createCommandReport(0, HostRoleStatus.FAILED,"{}",
String.format("Source type %s not found", YARN_SITE_CONFIG_TYPE), "");
}
-
- try {
- cluster.getService("SPARK"); // check if SPARK service present
Map<String, String> yarnSiteProperties = yarnSiteConfig.getProperties();
final List<String> auxSevices;
@@ -82,7 +79,11 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
} else {
auxSevices = new ArrayList<>();
}
- auxSevices.add(SPARK_SHUFFLE_AUX_STR);
+
+ // check if spark is not already in the list
+ if (!auxSevices.contains(SPARK_SHUFFLE_AUX_STR)) {
+ auxSevices.add(SPARK_SHUFFLE_AUX_STR);
+ }
newAuxServices = StringUtils.join(auxSevices, ",");
yarnSiteProperties.put(YARN_NODEMANAGER_AUX_SERVICES, newAuxServices);
@@ -94,10 +95,5 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
String.format("%s was set from %s to %s. %s was set to %s",
YARN_NODEMANAGER_AUX_SERVICES, oldAuxServices, newAuxServices,
YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE, YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE), "");
-
- } catch (ServiceNotFoundException e) {
- return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
- String.format("%s not updated as no SPARK service present on the cluster", YARN_NODEMANAGER_AUX_SERVICES), "");
- }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index ff7e032..c5d74e2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -395,7 +395,7 @@
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark Shuffle">
<task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index 34e4848..6b5bcb1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -566,7 +566,7 @@
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
- <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+ <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</pre-upgrade>
<upgrade>
<task xsi:type="restart-task"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index f58eb79..a3880b3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -269,7 +269,7 @@
<supports-auto-skip-failure>false</supports-auto-skip-failure>
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
- <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+ <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</execute-stage>
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 573713c..c15769b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -488,7 +488,7 @@
<component name="RESOURCEMANAGER">
<pre-upgrade>
- <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+ <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
index 134d9b0..5e8eae6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
@@ -20,26 +20,14 @@
<configuration supports_final="true">
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
+ <value>mapreduce_shuffle,spark_shuffle</value>
<description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
not start with numbers</description>
- <depends-on>
- <property>
- <type>spark-default</type>
- <name>spark.yarn.queue</name>
- </property>
- </depends-on>
</property>
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
<value>org.apache.spark.network.yarn.YarnShuffleService</value>
<description>The auxiliary service class to use for Spark</description>
- <depends-on>
- <property>
- <type>spark-default</type>
- <name>spark.yarn.queue</name>
- </property>
- </depends-on>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
index 3031ab3..985c101 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
@@ -19,80 +19,4 @@ limitations under the License.
class HDP24StackAdvisor(HDP23StackAdvisor):
- def getServiceConfigurationRecommenderDict(self):
- parent_recommend_conf_dict = super(HDP24StackAdvisor, self).getServiceConfigurationRecommenderDict()
- child_recommend_conf_dict = {
- "YARN": self.recommendYARNConfigurations
- }
- parent_recommend_conf_dict.update(child_recommend_conf_dict)
- return parent_recommend_conf_dict
-
- def getServiceConfigurationValidators(self):
- parentValidators = super(HDP24StackAdvisor, self).getServiceConfigurationValidators()
- childValidators = {
- "YARN": {"yarn-site": self.validateYARNConfigurations}
- }
- self.mergeValidators(parentValidators, childValidators)
- return parentValidators
-
- def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP24StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
-
- yarn_site_config = "yarn-site"
- properties = services["configurations"] if yarn_site_config in services["configurations"] else []
- yarn_site_properties = properties[yarn_site_config]["properties"] if yarn_site_config in properties and \
- "properties" in properties[yarn_site_config] else []
- put_yarn_site_property = self.putProperty(configurations, yarn_site_config, services)
- put_yarn_site_property_attributes = self.putPropertyAttribute(configurations, yarn_site_config)
- services_list = [service["StackServices"]["service_name"] for service in services["services"]]
-
- if 'SPARK' in services_list:
- if "yarn.nodemanager.aux-services" in yarn_site_properties:
- aux_services = yarn_site_properties["yarn.nodemanager.aux-services"].split(",")
- if "spark_shuffle" not in aux_services:
- aux_services.append("spark_shuffle")
- put_yarn_site_property("yarn.nodemanager.aux-services", ",".join(aux_services))
- else:
- put_yarn_site_property("yarn.nodemanager.aux-services", "spark_shuffle")
-
- put_yarn_site_property("yarn.nodemanager.aux-services.spark_shuffle.class",
- "org.apache.spark.network.yarn.YarnShuffleService")
- else:
- put_yarn_site_property_attributes("yarn.nodemanager.aux-services.spark_shuffle.class", "delete", "true")
-
- def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- yarn_site = properties
- validationItems = []
- services_list = [service["StackServices"]["service_name"] for service in services["services"]]
-
- if "SPARK" in services_list and 'YARN' in services_list:
- # yarn.nodemanager.aux-services = ...,spark_shuffle,....
- # yarn.nodemanager.aux-services.spark_shuffle.class = <not set>
- if "yarn.nodemanager.aux-services" in yarn_site \
- and "spark_shuffle" in yarn_site["yarn.nodemanager.aux-services"].lower() \
- and "yarn.nodemanager.aux-services.spark_shuffle.class" not in yarn_site:
- validationItems.append({
- "config-name": "yarn.nodemanager.aux-services.spark_shuffle.class",
- "item": self.getErrorItem("If spark_shuffle is listed in the aux-services, property value for " +
- "yarn.nodemanager.aux-services.spark_shuffle.class need to be set")
- })
-
- # yarn.nodemanager.aux-services = <not set>
- # yarn.nodemanager.aux-services.spark_shuffle.class = is set
- spark_aux_service_warning = False
- if "yarn.nodemanager.aux-services" in yarn_site and "spark_shuffle" not in yarn_site[
- "yarn.nodemanager.aux-services"].lower() \
- and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site:
- spark_aux_service_warning = True
-
- if "yarn.nodemanager.aux-services" not in yarn_site and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site:
- spark_aux_service_warning = True
-
- if spark_aux_service_warning:
- validationItems.append({
- "config-name": "yarn.nodemanager.aux-services",
- "item": self.getWarnItem("If yarn.nodemanager.aux-services.spark_shuffle.class is set, probably " +
- "aux-services property need to be updated to enable spark_shuffle")
- })
-
- return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+ pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
index edba6f0..e65a824 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
@@ -22,7 +22,6 @@ import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -30,7 +29,6 @@ import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Map;
-import org.apache.ambari.server.ServiceNotFoundException;
import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.agent.CommandReport;
@@ -39,7 +37,6 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigImpl;
-import org.apache.ambari.server.state.Service;
import org.easymock.EasyMock;
import org.junit.Before;
import org.junit.Test;
@@ -94,20 +91,16 @@ public class SparkShufflePropertyConfigTest {
}
@Test
- public void testSparkPresent() throws Exception {
+ public void testAction() throws Exception {
ExecutionCommand executionCommand = new ExecutionCommand();
Map<String, String> commandParams = new HashMap<>();
HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
- Service sparkService = EasyMock.createMock(Service.class);
SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
commandParams.put("clusterName", "c1");
executionCommand.setCommandParams(commandParams);
executionCommand.setClusterName("c1");
- // test calculations when spark service present
- expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce();
-
expect(hrc.getRequestId()).andReturn(1L).anyTimes();
expect(hrc.getStageId()).andReturn(2L).anyTimes();
expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
@@ -131,39 +124,5 @@ public class SparkShufflePropertyConfigTest {
assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class"));
}
- @Test
- public void testSparkNotPresent() throws Exception{
- SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
- Map<String, String> commandParams = new HashMap<>();
- ExecutionCommand executionCommand = new ExecutionCommand();
- HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
- ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK");
- commandParams.put("clusterName", "c1");
- executionCommand.setCommandParams(commandParams);
- executionCommand.setClusterName("c1");
-
- //no spark service present
- expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce();
-
- expect(hrc.getRequestId()).andReturn(1L).anyTimes();
- expect(hrc.getStageId()).andReturn(2L).anyTimes();
- expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
- replay(cluster, hrc);
-
- clusterField.set(action, m_clusters);
- action.setExecutionCommand(executionCommand);
- action.setHostRoleCommand(hrc);
-
- CommandReport report = action.execute(null);
- assertNotNull(report);
-
- Cluster c = m_clusters.getCluster("c1");
- Config config = c.getDesiredConfigByType("yarn-site");
- Map<String, String> map = config.getProperties();
-
- assertTrue(map.containsKey("yarn.nodemanager.aux-services"));
- assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class"));
- assertEquals("some_service", map.get("yarn.nodemanager.aux-services"));
- }
}
[10/19] ambari git commit: AMBARI-14678. RU - Oozie HA,
Service Check fails since the pre-upgrade directive has multiple
tasks that should run only on the specified hosts (alejandro)
Posted by nc...@apache.org.
AMBARI-14678. RU - Oozie HA, Service Check fails since the pre-upgrade directive has multiple tasks that should run only on the specified hosts (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cbed6402
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cbed6402
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cbed6402
Branch: refs/heads/branch-dev-patch-upgrade
Commit: cbed6402604d7e80a017c358a131191706255e16
Parents: f3ffe0c
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Mon Jan 18 11:41:37 2016 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Mon Jan 18 19:17:15 2016 -0800
----------------------------------------------------------------------
.../server/state/stack/upgrade/Grouping.java | 125 ++++++++-----
.../stack/upgrade/StageWrapperBuilder.java | 2 +-
.../ambari/server/state/stack/upgrade/Task.java | 8 +
.../server/state/stack/upgrade/TaskWrapper.java | 13 ++
.../state/stack/upgrade/TaskWrapperBuilder.java | 10 ++
.../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 5 +-
.../HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml | 29 +--
.../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 29 +--
.../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 29 +--
.../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 8 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 8 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 8 +-
.../HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml | 29 +--
.../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 29 +--
.../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 8 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 8 +-
.../HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml | 29 +--
.../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 8 +-
.../ambari/server/state/UpgradeHelperTest.java | 168 ++++++++++++++----
.../HDP/2.0.7/services/OOZIE/metainfo.xml | 176 +++++++++++++++++++
.../HDP/2.1.1/upgrades/upgrade_bucket_test.xml | 20 +--
.../upgrades/upgrade_nonrolling_new_stack.xml | 6 +-
.../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 45 ++++-
23 files changed, 520 insertions(+), 280 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
index aa852bd..5af893e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
@@ -100,61 +100,32 @@ public class Grouping {
boolean forUpgrade = ctx.getDirection().isUpgrade();
// Construct the pre tasks during Upgrade/Downgrade direction.
+ // Buckets are grouped by the type, e.g., bucket of all Execute tasks, or all Configure tasks.
List<TaskBucket> buckets = buckets(resolveTasks(forUpgrade, true, pc));
for (TaskBucket bucket : buckets) {
+ // The TaskWrappers take into account if a task is meant to run on all, any, or master.
+ // A TaskWrapper may contain multiple tasks, but typically only one, and they all run on the same set of hosts.
List<TaskWrapper> preTasks = TaskWrapperBuilder.getTaskList(service, pc.name, hostsType, bucket.tasks, params);
- Set<String> preTasksEffectiveHosts = TaskWrapperBuilder.getEffectiveHosts(preTasks);
- if (!preTasksEffectiveHosts.isEmpty()) {
- StageWrapper stage = new StageWrapper(
- bucket.type,
- getStageText("Preparing", ctx.getComponentDisplay(service, pc.name), preTasksEffectiveHosts),
- params,
- preTasks
- );
- m_stages.add(stage);
+ List<List<TaskWrapper>> organizedTasks = organizeTaskWrappersBySyncRules(preTasks);
+ for (List<TaskWrapper> tasks : organizedTasks) {
+ addTasksToStageInBatches(tasks, "Preparing", ctx, service, pc, params);
}
}
// Add the processing component
if (null != pc.tasks && 1 == pc.tasks.size()) {
Task t = pc.tasks.get(0);
- if(m_grouping.parallelScheduler != null) {
- List<Set<String>> hostSets = SetUtils.split(hostsType.hosts, m_grouping.parallelScheduler.maxDegreeOfParallelism);
- int batchNum = 1;
- for(Set<String> hosts : hostSets) {
- // Create single stage for all
- StageWrapper stage = new StageWrapper(
- t.getStageWrapperType(),
- getStageText(t.getActionVerb(), ctx.getComponentDisplay(service, pc.name), hosts, batchNum++, hostSets.size()),
- params,
- new TaskWrapper(service, pc.name, hosts, params, t));
- m_stages.add(stage);
- }
- } else {
- for (String hostName : hostsType.hosts) {
- StageWrapper stage = new StageWrapper(
- t.getStageWrapperType(),
- getStageText(t.getActionVerb(), ctx.getComponentDisplay(service, pc.name), Collections.singleton(hostName)),
- params,
- new TaskWrapper(service, pc.name, Collections.singleton(hostName), params, t));
- m_stages.add(stage);
- }
- }
+ TaskWrapper tw = new TaskWrapper(service, pc.name, hostsType.hosts, params, Collections.singletonList(t));
+ addTasksToStageInBatches(Collections.singletonList(tw), t.getActionVerb(), ctx, service, pc, params);
}
// Construct the post tasks during Upgrade/Downgrade direction.
buckets = buckets(resolveTasks(forUpgrade, false, pc));
for (TaskBucket bucket : buckets) {
List<TaskWrapper> postTasks = TaskWrapperBuilder.getTaskList(service, pc.name, hostsType, bucket.tasks, params);
- Set<String> postTasksEffectiveHosts = TaskWrapperBuilder.getEffectiveHosts(postTasks);
- if (!postTasksEffectiveHosts.isEmpty()) {
- StageWrapper stage = new StageWrapper(
- bucket.type,
- getStageText("Completing", ctx.getComponentDisplay(service, pc.name), postTasksEffectiveHosts),
- params,
- postTasks
- );
- m_stages.add(stage);
+ List<List<TaskWrapper>> organizedTasks = organizeTaskWrappersBySyncRules(postTasks);
+ for (List<TaskWrapper> tasks : organizedTasks) {
+ addTasksToStageInBatches(tasks, "Completing", ctx, service, pc, params);
}
}
@@ -165,6 +136,80 @@ public class Grouping {
}
/**
+ * Split a list of TaskWrappers into a list of lists where a TaskWrapper that has any task with isSequential == true
+ * must be a singleton in its own list.
+ * @param tasks List of TaskWrappers to analyze
+ * @return List of list of TaskWrappers, where each outer list is a separate stage.
+ */
+ private List<List<TaskWrapper>> organizeTaskWrappersBySyncRules(List<TaskWrapper> tasks) {
+ List<List<TaskWrapper>> groupedTasks = new ArrayList<List<TaskWrapper>>();
+
+ List<TaskWrapper> subTasks = new ArrayList<>();
+ for (TaskWrapper tw : tasks) {
+ // If an of this TaskWrapper's tasks must be on its own stage, write out the previous subtasks if possible into one complete stage.
+ if (tw.isAnyTaskSequential()) {
+ if (!subTasks.isEmpty()) {
+ groupedTasks.add(subTasks);
+ subTasks = new ArrayList<>();
+ }
+ groupedTasks.add(Collections.singletonList(tw));
+ } else {
+ subTasks.add(tw);
+ }
+ }
+
+ if (!subTasks.isEmpty()) {
+ groupedTasks.add(subTasks);
+ }
+
+ return groupedTasks;
+ }
+
+ /**
+ * Helper function to analyze a ProcessingComponent and add its task to stages, depending on the batch size.
+ * @param tasks Collection of tasks for this stage
+ * @param verb Verb string to use in the title of the task
+ * @param ctx Upgrade Context
+ * @param service Service
+ * @param pc Processing Component
+ * @param params Params to add to the stage.
+ */
+ private void addTasksToStageInBatches(List<TaskWrapper> tasks, String verb, UpgradeContext ctx, String service, ProcessingComponent pc, Map<String, String> params) {
+ if (tasks == null || tasks.isEmpty() || tasks.get(0).getTasks() == null || tasks.get(0).getTasks().isEmpty()) {
+ return;
+ }
+
+ // Our assumption is that all of the tasks in the StageWrapper are of the same type.
+ StageWrapper.Type type = tasks.get(0).getTasks().get(0).getStageWrapperType();
+
+ // Expand some of the TaskWrappers into multiple based on the batch size.
+ for (TaskWrapper tw : tasks) {
+ List<Set<String>> hostSets = null;
+ if (m_grouping.parallelScheduler != null && m_grouping.parallelScheduler.maxDegreeOfParallelism > 0) {
+ hostSets = SetUtils.split(tw.getHosts(), m_grouping.parallelScheduler.maxDegreeOfParallelism);
+ } else {
+ hostSets = SetUtils.split(tw.getHosts(), 1);
+ }
+
+ int numBatchesNeeded = hostSets.size();
+ int batchNum = 0;
+ for (Set<String> hostSubset : hostSets) {
+ batchNum++;
+ TaskWrapper expandedTW = new TaskWrapper(tw.getService(), tw.getComponent(), hostSubset, tw.getParams(), tw.getTasks());
+
+ String stageText = getStageText(verb, ctx.getComponentDisplay(service, pc.name), hostSubset, batchNum, numBatchesNeeded);
+
+ StageWrapper stage = new StageWrapper(
+ type,
+ stageText,
+ params,
+ new TaskWrapper(service, pc.name, hostSubset, params, tw.getTasks()));
+ m_stages.add(stage);
+ }
+ }
+ }
+
+ /**
* Determine if service checks need to be ran after the stages.
* @param upgradeContext the upgrade context
* @return Return the stages, which may potentially be followed by service checks.
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java
index b0df83f..e96c7ff 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java
@@ -169,7 +169,7 @@ public abstract class StageWrapperBuilder {
*/
protected String getStageText(String prefix, String component, Set<String> hosts, int batchNum, int totalBatches) {
String stageText = getStageText(prefix, component, hosts);
- String batchText = 1 == totalBatches? "" : String.format(" ( Batch %s of %s )", batchNum, totalBatches);
+ String batchText = 1 == totalBatches? "" : String.format(" (Batch %s of %s)", batchNum, totalBatches);
return stageText + batchText;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
index 980f88e..f365a7f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
@@ -17,6 +17,7 @@
*/
package org.apache.ambari.server.state.stack.upgrade;
+import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlSeeAlso;
@@ -34,6 +35,13 @@ public abstract class Task {
public String summary;
/**
+ * Whether the task needs to run sequentially, i.e., on its own stage.
+ * If false, will be grouped with other tasks.
+ */
+ @XmlAttribute(name = "sequential")
+ public boolean isSequential = false;
+
+ /**
* @return the type of the task
*/
public abstract Type getType();
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java
index 69b3f8b..5fdf91c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java
@@ -112,4 +112,17 @@ public class TaskWrapper {
return component;
}
+ /**
+ * @return true if any task is sequential, otherwise, return false.
+ */
+ public boolean isAnyTaskSequential() {
+ for (Task t : getTasks()) {
+ if (t.isSequential) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
index 057c310..81a3a4d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.ambari.server.stack.HostsType;
+import org.apache.ambari.server.utils.StageUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,8 +48,16 @@ public class TaskWrapperBuilder {
* @param params additional parameters
*/
public static List<TaskWrapper> getTaskList(String service, String component, HostsType hostsType, List<Task> tasks, Map<String, String> params) {
+ // Ok if Ambari Server is not part of the cluster hosts since this is only used in the calculation of how many batches
+ // to create.
+ String ambariServerHostname = StageUtils.getHostName();
+
List<TaskWrapper> collection = new ArrayList<TaskWrapper>();
for (Task t : tasks) {
+ if (t.getType().equals(Task.Type.CONFIGURE) || t.getType().equals(Task.Type.MANUAL)) {
+ collection.add(new TaskWrapper(service, component, Collections.singleton(ambariServerHostname), params, t));
+ continue;
+ }
if (t.getType().equals(Task.Type.EXECUTE)) {
ExecuteTask et = (ExecuteTask) t;
if (et.hosts == ExecuteHostType.MASTER) {
@@ -70,6 +79,7 @@ public class TaskWrapperBuilder {
continue;
}
}
+ // Otherwise, meant to run on ALL hosts.
}
collection.add(new TaskWrapper(service, component, hostsType.hosts, params, t));
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
index 2cf9da9..9f0458b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
@@ -793,14 +793,15 @@
from HDP 2.1->2.3.
-->
<task xsi:type="configure_function"/>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
index cbd9c2a..3aaf8c7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
@@ -415,31 +415,6 @@
</service>
</group>
- <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
- <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
- <direction>UPGRADE</direction>
- <skippable>true</skippable>
- <supports-auto-skip-failure>false</supports-auto-skip-failure>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>upgrade_oozie_database_and_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
- <!-- Only create the ShareLib folder during a Downgrade. -->
- <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
- <direction>DOWNGRADE</direction>
- <skippable>true</skippable>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>create_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
<group xsi:type="restart" name="OOZIE" title="Oozie">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -774,14 +749,14 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
index 816fe13..df12920 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
@@ -689,31 +689,6 @@
</service>
</group>
- <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
- <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
- <direction>UPGRADE</direction>
- <skippable>true</skippable>
- <supports-auto-skip-failure>false</supports-auto-skip-failure>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>upgrade_oozie_database_and_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
- <!-- Only create the ShareLib folder during a Downgrade. -->
- <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
- <direction>DOWNGRADE</direction>
- <skippable>true</skippable>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>create_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
<group xsi:type="restart" name="OOZIE" title="Oozie">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -1062,14 +1037,14 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index 5166c4b..ff7e032 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -753,31 +753,6 @@
</service>
</group>
- <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
- <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
- <direction>UPGRADE</direction>
- <skippable>true</skippable>
- <supports-auto-skip-failure>false</supports-auto-skip-failure>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>upgrade_oozie_database_and_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
- <!-- Only create the ShareLib folder during a Downgrade. -->
- <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
- <direction>DOWNGRADE</direction>
- <skippable>true</skippable>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>create_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
<group xsi:type="restart" name="OOZIE" title="Oozie">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -1153,14 +1128,14 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index 8b2f2a3..69dbe58 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -594,24 +594,24 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index 375da12..7e7138d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -741,24 +741,24 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index ea3d6a6..34e4848 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -753,24 +753,24 @@
<task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
<task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
index 34423f9..57b2f42 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
@@ -473,31 +473,6 @@
</service>
</group>
- <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
- <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
- <direction>UPGRADE</direction>
- <skippable>true</skippable>
- <supports-auto-skip-failure>false</supports-auto-skip-failure>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>upgrade_oozie_database_and_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
- <!-- Only create the ShareLib folder during a Downgrade. -->
- <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
- <direction>DOWNGRADE</direction>
- <skippable>true</skippable>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>create_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
<group xsi:type="restart" name="OOZIE" title="Oozie">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -879,14 +854,14 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index 50c8584..f58eb79 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -492,31 +492,6 @@
</service>
</group>
- <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
- <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
- <direction>UPGRADE</direction>
- <skippable>true</skippable>
- <supports-auto-skip-failure>false</supports-auto-skip-failure>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>upgrade_oozie_database_and_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
- <!-- Only create the ShareLib folder during a Downgrade. -->
- <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
- <direction>DOWNGRADE</direction>
- <skippable>true</skippable>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>create_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
<group xsi:type="restart" name="OOZIE" title="Oozie">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -898,14 +873,14 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
index 876342c..ab955e4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
@@ -689,24 +689,24 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index f145de1..573713c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -644,24 +644,24 @@
<pre-upgrade>
<task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml
index a9ee860..2600eae 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml
@@ -474,31 +474,6 @@
</service>
</group>
- <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
- <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
- <direction>UPGRADE</direction>
- <skippable>true</skippable>
- <supports-auto-skip-failure>false</supports-auto-skip-failure>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>upgrade_oozie_database_and_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
- <!-- Only create the ShareLib folder during a Downgrade. -->
- <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
- <direction>DOWNGRADE</direction>
- <skippable>true</skippable>
- <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
- <script>scripts/oozie_server_upgrade.py</script>
- <function>create_sharelib</function>
- </task>
- </execute-stage>
- </group>
-
<group xsi:type="restart" name="OOZIE" title="Oozie">
<service-check>false</service-check>
<skippable>true</skippable>
@@ -880,14 +855,14 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
index b8bd1ee..b91e27a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
@@ -687,24 +687,24 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
</pre-upgrade>
<pre-downgrade>
- <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
<script>scripts/oozie_server.py</script>
<function>stop</function>
</task>
- <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index 7e61942..eb5bf62 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -54,6 +54,8 @@ import org.apache.ambari.server.state.stack.UpgradePack;
import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition;
import org.apache.ambari.server.state.stack.upgrade.ConfigureTask;
import org.apache.ambari.server.state.stack.upgrade.Direction;
+import org.apache.ambari.server.state.stack.upgrade.ExecuteTask;
+import org.apache.ambari.server.state.stack.upgrade.Grouping;
import org.apache.ambari.server.state.stack.upgrade.ManualTask;
import org.apache.ambari.server.state.stack.upgrade.StageWrapper;
import org.apache.ambari.server.state.stack.upgrade.Task;
@@ -202,13 +204,14 @@ public class UpgradeHelperTest {
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
assertEquals("PRE_CLUSTER", groups.get(0).name);
assertEquals("ZOOKEEPER", groups.get(1).name);
assertEquals("CORE_MASTER", groups.get(2).name);
assertEquals("CORE_SLAVES", groups.get(3).name);
assertEquals("HIVE", groups.get(4).name);
+ assertEquals("OOZIE", groups.get(5).name);
UpgradeGroupHolder holder = groups.get(2);
boolean found = false;
@@ -238,7 +241,7 @@ public class UpgradeHelperTest {
assertTrue(mt.structuredOut.contains("DATANODE"));
assertTrue(mt.structuredOut.contains("NODEMANAGER"));
- UpgradeGroupHolder postGroup = groups.get(5);
+ UpgradeGroupHolder postGroup = groups.get(6);
assertEquals("POST_CLUSTER", postGroup.name);
assertEquals("Finalize Upgrade", postGroup.title);
assertEquals(3, postGroup.items.size());
@@ -249,7 +252,7 @@ public class UpgradeHelperTest {
assertEquals(4, groups.get(0).items.size());
assertEquals(6, groups.get(1).items.size());
- assertEquals(8, groups.get(2).items.size());
+ assertEquals(9, groups.get(2).items.size());
assertEquals(8, groups.get(3).items.size());
}
@@ -321,7 +324,7 @@ public class UpgradeHelperTest {
UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING);
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
for (UpgradeGroupHolder group : groups) {
for (StageWrapper stageWrapper : group.items) {
@@ -349,7 +352,7 @@ public class UpgradeHelperTest {
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
UpgradeGroupHolder mastersGroup = groups.get(2);
assertEquals("CORE_MASTER", mastersGroup.name);
@@ -400,14 +403,16 @@ public class UpgradeHelperTest {
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
assertEquals("PRE_CLUSTER", groups.get(0).name);
assertEquals("ZOOKEEPER", groups.get(1).name);
assertEquals("CORE_MASTER", groups.get(2).name);
assertEquals("CORE_SLAVES", groups.get(3).name);
+ assertEquals("HIVE", groups.get(4).name);
+ assertEquals("OOZIE", groups.get(5).name);
- UpgradeGroupHolder postGroup = groups.get(5);
+ UpgradeGroupHolder postGroup = groups.get(6);
assertEquals("POST_CLUSTER", postGroup.name);
assertEquals("Finalize Upgrade", postGroup.title);
assertEquals(3, postGroup.items.size());
@@ -417,7 +422,7 @@ public class UpgradeHelperTest {
assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType());
assertEquals(6, groups.get(1).items.size());
- assertEquals(8, groups.get(2).items.size());
+ assertEquals(9, groups.get(2).items.size());
assertEquals(7, groups.get(3).items.size());
}
@@ -435,17 +440,17 @@ public class UpgradeHelperTest {
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
- UpgradeGroupHolder preGroup = groups.get(0);
- assertEquals("PRE_CLUSTER", preGroup.name);
- assertEquals("HIVE", groups.get(1).name);
- assertEquals("CORE_SLAVES", groups.get(2).name);
- assertEquals("CORE_MASTER", groups.get(3).name);
- assertEquals("ZOOKEEPER", groups.get(4).name);
+ assertEquals("PRE_CLUSTER", groups.get(0).name);
+ assertEquals("OOZIE", groups.get(1).name);
+ assertEquals("HIVE", groups.get(2).name);
+ assertEquals("CORE_SLAVES", groups.get(3).name);
+ assertEquals("CORE_MASTER", groups.get(4).name);
+ assertEquals("ZOOKEEPER", groups.get(5).name);
- UpgradeGroupHolder postGroup = groups.get(5);
+ UpgradeGroupHolder postGroup = groups.get(6);
assertEquals("POST_CLUSTER", postGroup.name);
assertEquals("Finalize Downgrade", postGroup.title);
assertEquals(3, postGroup.items.size());
@@ -455,10 +460,10 @@ public class UpgradeHelperTest {
assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType());
assertEquals(4, groups.get(0).items.size());
- assertEquals(2, groups.get(1).items.size());
- assertEquals(8, groups.get(2).items.size());
- assertEquals(7, groups.get(3).items.size());
- assertEquals(5, groups.get(4).items.size());
+ assertEquals(8, groups.get(1).items.size());
+ assertEquals(3, groups.get(2).items.size());
+ assertEquals(8, groups.get(3).items.size());
+ assertEquals(8, groups.get(4).items.size());
}
@Test
@@ -478,7 +483,17 @@ public class UpgradeHelperTest {
assertEquals(1, groups.size());
UpgradeGroupHolder group = groups.iterator().next();
- assertEquals(6, group.items.size());
+ // Pre:
+ // Manual task = 1
+ // 2x - Execute task on all 3 = 6
+
+ // Post:
+ // Execute task on all 3 = 3
+ // 2x - Manual task = 2
+ // 3x - Execute task on all 3 = 9
+
+ // Service Check = 1
+ assertEquals(22, group.items.size());
}
@Test
@@ -495,7 +510,7 @@ public class UpgradeHelperTest {
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
// grab the manual task out of ZK which has placeholder text
UpgradeGroupHolder zookeeperGroup = groups.get(1);
@@ -523,13 +538,12 @@ public class UpgradeHelperTest {
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
// grab the configure task out of Hive
UpgradeGroupHolder hiveGroup = groups.get(4);
assertEquals("HIVE", hiveGroup.name);
- ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(
- 1).getTasks().get(0);
+ ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0);
// now change the thrift port to http to have the 2nd condition invoked
Map<String, String> hiveConfigs = new HashMap<String, String>();
@@ -581,10 +595,8 @@ public class UpgradeHelperTest {
assertEquals("delete-if-key-present", transfers.get(8).deleteKey);
assertEquals("delete-if-key-absent", transfers.get(9).deleteKey);
-
}
-
@Test
public void testConfigureTask() throws Exception {
Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
@@ -601,7 +613,7 @@ public class UpgradeHelperTest {
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade,
context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
// grab the configure task out of Hive
UpgradeGroupHolder hiveGroup = groups.get(4);
@@ -674,12 +686,12 @@ public class UpgradeHelperTest {
List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
- assertEquals(6, groups.size());
+ assertEquals(7, groups.size());
// grab the configure task out of Hive
UpgradeGroupHolder hiveGroup = groups.get(4);
assertEquals("HIVE", hiveGroup.name);
- ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(1).getTasks().get(0);
+ ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0);
Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup);
assertFalse(configProperties.isEmpty());
@@ -754,7 +766,7 @@ public class UpgradeHelperTest {
UpgradeGroupHolder holder = groups.get(3);
assertEquals(holder.name, "SERVICE_CHECK_1");
- assertEquals(6, holder.items.size());
+ assertEquals(7, holder.items.size());
int numServiceChecksActual = 0;
for (StageWrapper sw : holder.items) {
for(Service service : services) {
@@ -881,7 +893,7 @@ public class UpgradeHelperTest {
assertEquals(4, groups.get(0).items.size());
assertEquals(5, groups.get(1).items.size());
- assertEquals(8, groups.get(2).items.size());
+ assertEquals(9, groups.get(2).items.size());
assertEquals(8, groups.get(3).items.size());
}
@@ -934,6 +946,7 @@ public class UpgradeHelperTest {
c.addService(serviceFactory.createNew(c, "YARN"));
c.addService(serviceFactory.createNew(c, "ZOOKEEPER"));
c.addService(serviceFactory.createNew(c, "HIVE"));
+ c.addService(serviceFactory.createNew(c, "OOZIE"));
Service s = c.getService("HDFS");
ServiceComponent sc = s.addServiceComponent("NAMENODE");
@@ -962,6 +975,16 @@ public class UpgradeHelperTest {
sc = s.addServiceComponent("HIVE_SERVER");
sc.addServiceComponentHost("h2");
+ s = c.getService("OOZIE");
+ // Oozie Server HA
+ sc = s.addServiceComponent("OOZIE_SERVER");
+ sc.addServiceComponentHost("h2");
+ sc.addServiceComponentHost("h3");
+ sc = s.addServiceComponent("OOZIE_CLIENT");
+ sc.addServiceComponentHost("h1");
+ sc.addServiceComponentHost("h2");
+ sc.addServiceComponentHost("h3");
+
// set some desired configs
Map<String, String> hiveConfigs = new HashMap<String, String>();
hiveConfigs.put("hive.server2.transport.mode", "binary");
@@ -1007,12 +1030,23 @@ public class UpgradeHelperTest {
expect(m_masterHostResolver.getMasterAndHosts("YARN", "RESOURCEMANAGER")).andReturn(type).anyTimes();
type = new HostsType();
+ expect(m_masterHostResolver.getMasterAndHosts("YARN", "APP_TIMELINE_SERVER")).andReturn(type).anyTimes();
+
+ type = new HostsType();
type.hosts.addAll(Arrays.asList("h1", "h3"));
expect(m_masterHostResolver.getMasterAndHosts("YARN", "NODEMANAGER")).andReturn(type).anyTimes();
expect(m_masterHostResolver.getMasterAndHosts("HIVE", "HIVE_SERVER")).andReturn(
type).anyTimes();
+ type = new HostsType();
+ type.hosts.addAll(Arrays.asList("h2", "h3"));
+ expect(m_masterHostResolver.getMasterAndHosts("OOZIE", "OOZIE_SERVER")).andReturn(type).anyTimes();
+
+ type = new HostsType();
+ type.hosts.addAll(Arrays.asList("h1", "h2", "h3"));
+ expect(m_masterHostResolver.getMasterAndHosts("OOZIE", "OOZIE_CLIENT")).andReturn(type).anyTimes();
+
expect(m_masterHostResolver.getCluster()).andReturn(c).anyTimes();
replay(m_masterHostResolver);
@@ -1020,6 +1054,76 @@ public class UpgradeHelperTest {
return c;
}
+ /**
+ * Test that multiple execute tasks with an annotation of synchronized="true" each run in their own stage.
+ */
+ @Test
+ public void testUpgradeWithMultipleTasksInOwnStage() throws Exception {
+ Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
+
+ assertTrue(upgrades.containsKey("upgrade_test"));
+ UpgradePack upgrade = upgrades.get("upgrade_test");
+ assertNotNull(upgrade);
+ assertTrue(upgrade.getType() == UpgradeType.ROLLING);
+
+ List<Grouping> upgradePackGroups = upgrade.getGroups(Direction.UPGRADE);
+
+ boolean foundService = false;
+ for (Grouping group : upgradePackGroups) {
+ if (group.title.equals("Oozie")) {
+ foundService = true;
+ }
+ }
+ assertTrue(foundService);
+
+ makeCluster();
+
+ UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21,
+ HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING);
+
+ List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
+
+ // The upgrade pack has 2 tasks for Oozie in the pre-upgrade group.
+ // The first task runs on "all", i.e., both Oozie Servers, whereas the
+ // second task runs on "any", i.e., exactly one.
+ int numPrepareStages = 0;
+ for (UpgradeGroupHolder group : groups) {
+ if (group.name.equals("OOZIE")) {
+ assertTrue(group.items.size() > 0);
+ for (StageWrapper sw : group.items) {
+
+ if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2 (Batch 1 of 2)") ||
+ sw.getText().equalsIgnoreCase("Preparing Oozie Server on h3 (Batch 2 of 2)")) {
+ numPrepareStages++;
+ List<TaskWrapper> taskWrappers = sw.getTasks();
+ assertEquals(1, taskWrappers.size());
+ List<Task> tasks = taskWrappers.get(0).getTasks();
+ assertEquals(1, taskWrappers.get(0).getHosts().size());
+ assertEquals(1, tasks.size());
+
+ ExecuteTask task = (ExecuteTask) tasks.get(0);
+ assertTrue("scripts/oozie_server.py".equalsIgnoreCase(task.script));
+ assertTrue("stop".equalsIgnoreCase(task.function));
+ }
+
+ if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2")) {
+ numPrepareStages++;
+ List<TaskWrapper> taskWrappers = sw.getTasks();
+ assertEquals(1, taskWrappers.size());
+ List<Task> tasks = taskWrappers.get(0).getTasks();
+ assertEquals(1, taskWrappers.get(0).getHosts().size());
+ assertEquals(1, tasks.size());
+
+ ExecuteTask task = (ExecuteTask) tasks.get(0);
+ assertTrue("scripts/oozie_server_upgrade.py".equalsIgnoreCase(task.script));
+ assertTrue("upgrade_oozie_database_and_sharelib".equalsIgnoreCase(task.function));
+ }
+ }
+ }
+ }
+ assertEquals(3, numPrepareStages);
+ }
+
@Test
public void testDowngradeAfterPartialUpgrade() throws Exception {
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml
new file mode 100644
index 0000000..6010659
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml
@@ -0,0 +1,176 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>OOZIE</name>
+ <displayName>Oozie</displayName>
+ <comment>System for workflow coordination and execution of Apache Hadoop jobs. This also includes the installation of the optional Oozie Web Console which relies on and will install the <a target="_blank" href="http://www.sencha.com/legal/open-source-faq/">ExtJS</a> Library.</comment>
+ <version>4.2.0.2.3</version>
+
+ <components>
+ <component>
+ <name>OOZIE_SERVER</name>
+ <displayName>Oozie Server</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>YARN/YARN_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/oozie_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>OOZIE_CLIENT</name>
+ <displayName>Oozie Client</displayName>
+ <category>CLIENT</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/oozie_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>oozie-site.xml</fileName>
+ <dictionaryName>oozie-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>oozie-env.sh</fileName>
+ <dictionaryName>oozie-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>oozie-log4j.properties</fileName>
+ <dictionaryName>oozie-log4j</dictionaryName>
+ </configFile>
+ </configFiles>
+ </component>
+ </components>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>oozie-site</config-type>
+ <config-type>oozie-env</config-type>
+ <config-type>oozie-log4j</config-type>
+ <config-type>yarn-site</config-type>
+ <config-type>hive-site</config-type>
+ <config-type>tez-site</config-type>
+ </configuration-dependencies>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>zip</name>
+ </package>
+ <package>
+ <name>mysql-connector-java</name>
+ <skipUpgrade>true</skipUpgrade>
+ </package>
+ <package>
+ <name>extjs</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat5,redhat6,redhat7,suse11</osFamily>
+ <packages>
+ <package>
+ <name>oozie_2_3_*</name>
+ </package>
+ <package>
+ <name>falcon_2_3_*</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
+ <packages>
+ <package>
+ <name>oozie-2-3-.*</name>
+ </package>
+ <package>
+ <name>falcon-2-3-.*</name>
+ </package>
+ <package>
+ <name>extjs</name>
+ </package>
+ <package>
+ <name>libxml2-utils</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ </service>
+ </services>
+</metainfo>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml
index 0e6d914..085ba25 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml
@@ -46,36 +46,34 @@
<component name="ZOOKEEPER_SERVER">
<pre-upgrade>
<task xsi:type="manual">
- <message>this is pre</message>
+ <message>pre-manual 1 task to run on all hosts</message>
</task>
<task xsi:type="execute">
- <command>ls</command>
+ <command>pre-ls 1 to run on all hosts</command>
</task>
<task xsi:type="execute">
- <command>ls</command>
+ <command>pre-ls 2 to run on all hosts</command>
</task>
-
</pre-upgrade>
<post-upgrade>
<task xsi:type="execute">
- <command>ls</command>
+ <command>post-ls 1 to run on all hosts</command>
</task>
<task xsi:type="manual">
- <message>this is pre</message>
+ <message>post-manual 1 task to run on all hosts</message>
</task>
<task xsi:type="manual">
- <message>this is pre</message>
+ <message>post-manual 2 task to run on all hosts</message>
</task>
<task xsi:type="execute">
- <command>ls</command>
+ <command>post-ls 2 to run on all hosts</command>
</task>
<task xsi:type="execute">
- <command>ls</command>
+ <command>post-ls 3 to run on all hosts</command>
</task>
-
<task xsi:type="execute">
- <command>ls</command>
+ <command>post-ls 4 to run on all hosts</command>
</task>
</post-upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml
index 8f78d0f..c79c8e7 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml
@@ -442,7 +442,7 @@
<skippable>true</skippable>
<supports-auto-skip-failure>false</supports-auto-skip-failure>
<execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
@@ -454,7 +454,7 @@
<direction>DOWNGRADE</direction>
<skippable>true</skippable>
<execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>create_sharelib</function>
</task>
@@ -809,7 +809,7 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
<pre-upgrade>
- <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib">
<script>scripts/oozie_server_upgrade.py</script>
<function>upgrade_oozie_database_and_sharelib</function>
</task>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
index 4f71b4a..237da77 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
@@ -101,7 +101,17 @@
<component>HIVE_SERVER</component>
<component>WEBHCAT_SERVER</component>
</service>
- </group>
+ </group>
+
+ <group name="OOZIE" title="Oozie">
+ <skippable>true</skippable>
+ <supports-auto-skip-failure>false</supports-auto-skip-failure>
+ <service-check>false</service-check>
+ <service name="OOZIE">
+ <component>OOZIE_SERVER</component>
+ <component>OOZIE_CLIENT</component>
+ </service>
+ </group>
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
<execute-stage title="Confirm Finalize">
@@ -199,13 +209,38 @@
<summary>HiveServer Port Availability</summary>
<message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message>
</task>
-
<task xsi:type="configure" id="hdp_2_1_1_set_transport_mode"/>
-
<task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/>
-
</pre-upgrade>
</component>
- </service>
+ </service>
+
+ <service name="OOZIE">
+ <component name="OOZIE_SERVER">
+ <pre-upgrade>
+ <!-- This is important, do not remove it since UpgradeHelperTest.java :
+ testUpgradeWithMultipleTasksWithMultipleHostTypes() asserts
+ that these tasks each run on their own stage. -->
+ <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
+ <script>scripts/oozie_server.py</script>
+ <function>stop</function>
+ </task>
+
+ <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
+ <script>scripts/oozie_server_upgrade.py</script>
+ <function>upgrade_oozie_database_and_sharelib</function>
+ </task>
+ </pre-upgrade>
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+
+ <component name="OOZIE_CLIENT">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
</processing>
</upgrade>
[05/19] ambari git commit: AMBARI-14665. Race condition problem in
ServiceComponentHostImpl when provisioning cluster with Blueprint (Sandor
Magyari via rlevas)
Posted by nc...@apache.org.
AMBARI-14665. Race condition problem in ServiceComponentHostImpl when provisioning cluster with Blueprint (Sandor Magyari via rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0cbefb3e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0cbefb3e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0cbefb3e
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 0cbefb3e6ceebea32d71461152aa1f141a8ed5d6
Parents: 36af9bb
Author: Sandor Magyari <sm...@hortonworks.com>
Authored: Mon Jan 18 13:35:32 2016 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Mon Jan 18 13:36:02 2016 -0500
----------------------------------------------------------------------
.../svccomphost/ServiceComponentHostImpl.java | 65 +++++++++++++++-----
1 file changed, 49 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0cbefb3e/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index 7bc4680..bfb6214 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -821,7 +821,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
HostComponentStateEntity stateEntity = getStateEntity();
if (stateEntity != null) {
getStateEntity().setCurrentState(state);
- saveIfPersisted();
+ saveComponentStateEntityIfPersisted();
} else {
LOG.warn("Setting a member on an entity object that may have been " +
"previously deleted, serviceName = " + getServiceName() + ", " +
@@ -860,7 +860,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
HostComponentStateEntity stateEntity = getStateEntity();
if (stateEntity != null) {
getStateEntity().setVersion(version);
- saveIfPersisted();
+ saveComponentStateEntityIfPersisted();
} else {
LOG.warn("Setting a member on an entity object that may have been " +
"previously deleted, serviceName = " + getServiceName() + ", " +
@@ -899,7 +899,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
HostComponentStateEntity stateEntity = getStateEntity();
if (stateEntity != null) {
getStateEntity().setSecurityState(securityState);
- saveIfPersisted();
+ saveComponentStateEntityIfPersisted();
} else {
LOG.warn("Setting a member on an entity object that may have been " +
"previously deleted, serviceName = " + getServiceName() + ", " +
@@ -940,8 +940,11 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
writeLock.lock();
try {
+ LOG.debug("Set DesiredSecurityState on serviceName = {} componentName = {} hostName = {} to {}",
+ getServiceName(), getServiceComponentName(), getHostName(), securityState);
+
getDesiredStateEntity().setSecurityState(securityState);
- saveIfPersisted();
+ saveComponentDesiredStateEntityIfPersisted();
} finally {
writeLock.unlock();
}
@@ -963,7 +966,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
HostComponentStateEntity stateEntity = getStateEntity();
if (stateEntity != null) {
stateEntity.setUpgradeState(upgradeState);
- saveIfPersisted();
+ saveComponentStateEntityIfPersisted();
} else {
LOG.warn("Setting a member on an entity object that may have been " +
"previously deleted, serviceName = " + getServiceName() + ", " +
@@ -1015,7 +1018,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
try {
stateMachine.doTransition(event.getType(), event);
getStateEntity().setCurrentState(stateMachine.getCurrentState());
- saveIfPersisted();
+ saveComponentStateEntityIfPersisted();
// TODO Audit logs
} catch (InvalidStateTransitionException e) {
LOG.error("Can't handle ServiceComponentHostEvent event at"
@@ -1175,7 +1178,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
HostComponentStateEntity stateEntity = getStateEntity();
if (stateEntity != null) {
stateEntity.setCurrentStack(stackEntity);
- saveIfPersisted();
+ saveComponentStateEntityIfPersisted();
} else {
LOG.warn("Setting a member on an entity object that may have been " +
"previously deleted, serviceName = " + getServiceName() + ", " +
@@ -1212,10 +1215,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
public void setDesiredState(State state) {
writeLock.lock();
try {
+ LOG.debug("Set DesiredState on serviceName = {} componentName = {} hostName = {} to {} ",
+ getServiceName(), getServiceComponentName(), getHostName(), state);
+
HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
if (desiredStateEntity != null) {
desiredStateEntity.setDesiredState(state);
- saveIfPersisted();
+ saveComponentDesiredStateEntityIfPersisted();
} else {
LOG.warn("Setting a member on an entity object that may have been " +
"previously deleted, serviceName = " + getServiceName() + ", " +
@@ -1253,13 +1259,16 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
public void setDesiredStackVersion(StackId stackId) {
writeLock.lock();
try {
+ LOG.debug("Set DesiredStackVersion on serviceName = {} componentName = {} hostName = {} to {}",
+ getServiceName(), getServiceComponentName(), getHostName(), stackId);
+
HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
if (desiredStateEntity != null) {
StackEntity stackEntity = stackDAO.find(stackId.getStackName(),
stackId.getStackVersion());
desiredStateEntity.setDesiredStack(stackEntity);
- saveIfPersisted();
+ saveComponentDesiredStateEntityIfPersisted();
}
} finally {
writeLock.unlock();
@@ -1290,10 +1299,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
public void setComponentAdminState(HostComponentAdminState attribute) {
writeLock.lock();
try {
+ LOG.debug("Set ComponentAdminState on serviceName = {} componentName = {} hostName = {} to {}",
+ getServiceName(), getServiceComponentName(), getHostName(), attribute);
+
HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
if (desiredStateEntity != null) {
desiredStateEntity.setAdminState(attribute);
- saveIfPersisted();
+ saveComponentDesiredStateEntityIfPersisted();
} else {
LOG.warn("Setting a member on an entity object that may have been " +
"previously deleted, serviceName = " + getServiceName() + ", " +
@@ -1426,7 +1438,8 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
eventPublisher.publish(event);
} else {
- saveIfPersisted();
+ saveComponentStateEntityIfPersisted();
+ saveComponentDesiredStateEntityIfPersisted();
}
} finally {
writeLock.unlock();
@@ -1481,19 +1494,33 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
}
/**
- * Merges the encapsulated {@link HostComponentStateEntity} and
- * {@link HostComponentDesiredStateEntity} inside of a new transaction. This
+ * Merges the encapsulated {@link HostComponentStateEntity} inside of a new transaction. This
* method assumes that the appropriate write lock has already been acquired
* from {@link #readWriteLock}.
*/
@Transactional
- void saveIfPersisted() {
+ void saveComponentStateEntityIfPersisted() {
if (isPersisted()) {
hostComponentStateDAO.merge(stateEntity);
+ }
+ }
+
+ /**
+ * Merges the encapsulated {@link HostComponentDesiredStateEntity} inside of a new transaction. This
+ * method assumes that the appropriate write lock has already been acquired
+ * from {@link #readWriteLock}.
+ */
+ @Transactional
+ void saveComponentDesiredStateEntityIfPersisted() {
+ if (isPersisted()) {
+ LOG.debug("Save desiredStateEntity serviceName = {} componentName = {} hostName = {} desiredState = {}",
+ getServiceName(), getServiceComponentName(), getHostName(), desiredStateEntity.getDesiredState());
+
hostComponentDesiredStateDAO.merge(desiredStateEntity);
}
}
+
@Override
public boolean canBeRemoved() {
clusterGlobalLock.readLock().lock();
@@ -1645,10 +1672,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
public void setMaintenanceState(MaintenanceState state) {
writeLock.lock();
try {
+ LOG.debug("Set MaintenanceState on serviceName = {} componentName = {} hostName = {} to {}",
+ getServiceName(), getServiceComponentName(), getHostName(), state);
+
HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
if (desiredStateEntity != null) {
desiredStateEntity.setMaintenanceState(state);
- saveIfPersisted();
+ saveComponentDesiredStateEntityIfPersisted();
// broadcast the maintenance mode change
MaintenanceModeEvent event = new MaintenanceModeEvent(state, this);
@@ -1708,10 +1738,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
public void setRestartRequired(boolean restartRequired) {
writeLock.lock();
try {
+ LOG.debug("Set RestartRequired on serviceName = {} componentName = {} hostName = {} to {}",
+ getServiceName(), getServiceComponentName(), getHostName(), restartRequired);
+
HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
if (desiredStateEntity != null) {
desiredStateEntity.setRestartRequired(restartRequired);
- saveIfPersisted();
+ saveComponentDesiredStateEntityIfPersisted();
helper.invalidateStaleConfigsCache(this);
} else {
LOG.warn("Setting a member on an entity object that may have been " +
[09/19] ambari git commit: AMBARI-14698. Kerberos automation logic
should use stack advisor when determining configuration updates (rlevas)
Posted by nc...@apache.org.
AMBARI-14698. Kerberos automation logic should use stack advisor when determining configuration updates (rlevas)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f3ffe0c2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f3ffe0c2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f3ffe0c2
Branch: refs/heads/branch-dev-patch-upgrade
Commit: f3ffe0c2094bd619cffbdd2daf4a0aa7e033f128
Parents: 132c535
Author: Robert Levas <rl...@hortonworks.com>
Authored: Mon Jan 18 20:01:29 2016 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Mon Jan 18 20:01:29 2016 -0500
----------------------------------------------------------------------
.../server/controller/KerberosHelper.java | 41 +++-
.../server/controller/KerberosHelperImpl.java | 210 +++++++++++++++++--
.../BlueprintConfigurationProcessor.java | 5 -
.../serveraction/ServerActionExecutor.java | 8 +
.../AbstractPrepareKerberosServerAction.java | 90 +++++---
.../PrepareDisableKerberosServerAction.java | 2 +-
.../PrepareEnableKerberosServerAction.java | 2 +-
.../PrepareKerberosIdentitiesServerAction.java | 3 +-
.../topology/ClusterConfigurationRequest.java | 17 +-
.../stacks/HDP/2.0.6/services/stack_advisor.py | 2 +-
.../StackAdvisorBlueprintProcessorTest.java | 3 +
.../server/controller/KerberosHelperTest.java | 46 +++-
.../ClusterConfigurationRequestTest.java | 3 +-
13 files changed, 362 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index b6686b4..5008d60 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -224,6 +224,8 @@ public interface KerberosHelper {
* @param cluster the cluster
* @param existingConfigurations the cluster's existing configurations
* @param services the set of services to process
+ * @param kerberosEnabled true if kerberos is (to be) enabled; otherwise false
+ * @param applyStackAdvisorUpdates true to invoke the stack advisor to validate property updates; false to skip
* @return a map of configuration updates
* @throws AmbariException
* @throws KerberosInvalidConfigurationException if an issue occurs trying to get the
@@ -231,13 +233,37 @@ public interface KerberosHelper {
*/
Map<String, Map<String, String>> getServiceConfigurationUpdates(Cluster cluster,
Map<String, Map<String, String>> existingConfigurations,
- Set<String> services, boolean serviceAlreadyExists)
+ Set<String> services,
+ boolean serviceAlreadyExists,
+ boolean kerberosEnabled,
+ boolean applyStackAdvisorUpdates)
throws KerberosInvalidConfigurationException, AmbariException;
/**
- * Ensures that the relevant headless (or user) Kerberos identities are created and cached.
+ * Invokes the Stack Advisor to help determine relevant configuration changes when enabling or
+ * disabling Kerberos
*
+ * @param cluster a cluster
+ * @param services a set of services that are being configured to enabled or disable Kerberos
+ * @param existingConfigurations the cluster's existing configurations
+ * @param kerberosConfigurations the configuration updates to make (must not be mutable)
+ * @param propertiesToIgnore the configuration properties that should be ignored when applying recommendations
+ * @param kerberosEnabled true if kerberos is (to be) enabled; otherwise false
+ * @return the configuration updates
+ * @throws AmbariException
+ */
+ Map<String, Map<String, String>> applyStackAdvisorUpdates(Cluster cluster, Set<String> services,
+ Map<String, Map<String, String>> existingConfigurations,
+ Map<String, Map<String, String>> kerberosConfigurations,
+ Map<String, Set<String>> propertiesToIgnore,
+ boolean kerberosEnabled)
+ throws AmbariException;
+
+ /**
+ * Ensures that the relevant headless (or user) Kerberos identities are created and cached.
+ * <p/>
* This can be called any number of times and only the missing identities will be created.
+ *
* @param cluster the cluster
* @param existingConfigurations the cluster's existing configurations
* @param services the set of services to process
@@ -453,6 +479,17 @@ public interface KerberosHelper {
Boolean getManageIdentitiesDirective(Map<String, String> requestProperties);
/**
+ * Given a list of KerberosIdentityDescriptors, returns a Map fo configuration types to property
+ * names and values.
+ * <p/>
+ * The property names and values are not expected to have any variable replacements done.
+ *
+ * @param identityDescriptors a List of KerberosIdentityDescriptor from which to retrieve configurations
+ * @return a Map of configuration types to property name/value pairs (as a Map)
+ */
+ Map<String, Map<String, String>> getIdentityConfigurations(List<KerberosIdentityDescriptor> identityDescriptors);
+
+ /**
* Returns the active identities for the named cluster. Results are filtered by host, service,
* and/or component; and grouped by host.
* <p/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index 6f0fa27..019fb3b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -30,6 +30,9 @@ import org.apache.ambari.server.actionmanager.RequestFactory;
import org.apache.ambari.server.actionmanager.Stage;
import org.apache.ambari.server.actionmanager.StageFactory;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import org.apache.ambari.server.configuration.Configuration;
import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
import org.apache.ambari.server.controller.internal.RequestImpl;
@@ -204,6 +207,9 @@ public class KerberosHelperImpl implements KerberosHelper {
@Inject
private CredentialStoreService credentialStoreService;
+ @Inject
+ private StackAdvisorHelper stackAdvisorHelper;
+
/**
* Used to get kerberos descriptors associated with the cluster or stack.
* Currently not available via injection.
@@ -310,7 +316,7 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, Map<String, String>> existingConfigurations = calculateExistingConfigurations(cluster, hostName);
Map<String, Map<String, String>> updates = getServiceConfigurationUpdates(cluster,
- existingConfigurations, Collections.singleton(serviceName), serviceAlreadyExists(cluster, serviceComponentHost));
+ existingConfigurations, Collections.singleton(serviceName), serviceAlreadyExists(cluster, serviceComponentHost), true, true);
for (Map.Entry<String, Map<String, String>> entry : updates.entrySet()) {
configHelper.updateConfigType(cluster, ambariManagementController, entry.getKey(), entry.getValue(), null,
@@ -335,8 +341,12 @@ public class KerberosHelperImpl implements KerberosHelper {
}
@Override
- public Map<String, Map<String, String>> getServiceConfigurationUpdates(Cluster cluster, Map<String, Map<String, String>> existingConfigurations, Set<String> services,
- boolean serviceAlreadyExists)
+ public Map<String, Map<String, String>> getServiceConfigurationUpdates(Cluster cluster,
+ Map<String, Map<String, String>> existingConfigurations,
+ Set<String> services,
+ boolean serviceAlreadyExists,
+ boolean kerberosEnabled,
+ boolean applyStackAdvisorUpdates)
throws KerberosInvalidConfigurationException, AmbariException {
Map<String, Map<String, String>> kerberosConfigurations = new HashMap<String, Map<String, String>>();
@@ -347,6 +357,8 @@ public class KerberosHelperImpl implements KerberosHelper {
Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
+ Map<String, Set<String>> propertiesToIgnore = new HashMap<String, Set<String>>();
+
for (String serviceName : services) {
// Set properties...
KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(serviceName);
@@ -356,21 +368,42 @@ public class KerberosHelperImpl implements KerberosHelper {
for (KerberosComponentDescriptor componentDescriptor : componentDescriptors.values()) {
if (componentDescriptor != null) {
Map<String, Map<String, String>> identityConfigurations;
- List<KerberosIdentityDescriptor> identities;
- identities = serviceDescriptor.getIdentities(true);
- identityConfigurations = getConfigurations(identities);
+ identityConfigurations = getIdentityConfigurations(serviceDescriptor.getIdentities(true));
if (identityConfigurations != null) {
for (Map.Entry<String, Map<String, String>> entry : identityConfigurations.entrySet()) {
- mergeConfigurations(kerberosConfigurations, entry.getKey(), entry.getValue(), configurations);
+ String configType = entry.getKey();
+ Map<String, String> properties = entry.getValue();
+
+ mergeConfigurations(kerberosConfigurations, configType, entry.getValue(), configurations);
+
+ if ((properties != null) && !properties.isEmpty()) {
+ Set<String> propertyNames = propertiesToIgnore.get(configType);
+ if (propertyNames == null) {
+ propertyNames = new HashSet<String>();
+ propertiesToIgnore.put(configType, propertyNames);
+ }
+ propertyNames.addAll(properties.keySet());
+ }
}
}
- identities = componentDescriptor.getIdentities(true);
- identityConfigurations = getConfigurations(identities);
+ identityConfigurations = getIdentityConfigurations(componentDescriptor.getIdentities(true));
if (identityConfigurations != null) {
for (Map.Entry<String, Map<String, String>> entry : identityConfigurations.entrySet()) {
- mergeConfigurations(kerberosConfigurations, entry.getKey(), entry.getValue(), configurations);
+ String configType = entry.getKey();
+ Map<String, String> properties = entry.getValue();
+
+ mergeConfigurations(kerberosConfigurations, configType, entry.getValue(), configurations);
+
+ if ((properties != null) && !properties.isEmpty()) {
+ Set<String> propertyNames = propertiesToIgnore.get(configType);
+ if (propertyNames == null) {
+ propertyNames = new HashSet<String>();
+ propertiesToIgnore.put(configType, propertyNames);
+ }
+ propertyNames.addAll(properties.keySet());
+ }
}
}
@@ -383,6 +416,151 @@ public class KerberosHelperImpl implements KerberosHelper {
setAuthToLocalRules(kerberosDescriptor, cluster, kerberosDetails.getDefaultRealm(), configurations, kerberosConfigurations);
+ return (applyStackAdvisorUpdates)
+ ? applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations, propertiesToIgnore, kerberosEnabled)
+ : kerberosConfigurations;
+ }
+
+ public Map<String, Map<String, String>> applyStackAdvisorUpdates(Cluster cluster, Set<String> services,
+ Map<String, Map<String, String>> existingConfigurations,
+ Map<String, Map<String, String>> kerberosConfigurations,
+ Map<String, Set<String>> propertiesToIgnore,
+ boolean kerberosEnabled) throws AmbariException {
+
+ StackId stackVersion = cluster.getCurrentStackVersion();
+
+ List<String> hostNames = new ArrayList<String>();
+ Collection<Host> hosts = cluster.getHosts();
+
+ if (hosts != null) {
+ for (Host host : hosts) {
+ hostNames.add(host.getHostName());
+ }
+ }
+
+ // Don't actually call the stack advisor if no hosts are in the cluster, else the stack advisor
+ // will throw a StackAdvisorException stating "Hosts and services must not be empty".
+ // This could happen when enabling Kerberos while installing a cluster via Blueprints due to the
+ // way hosts are discovered during the install process.
+ if(!hostNames.isEmpty()) {
+ Map<String, Map<String, Map<String, String>>> requestConfigurations = new HashMap<String, Map<String, Map<String, String>>>();
+ if (existingConfigurations != null) {
+ for (Map.Entry<String, Map<String, String>> configuration : existingConfigurations.entrySet()) {
+ Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+ String configType = configuration.getKey();
+ Map<String, String> configurationProperties = configuration.getValue();
+
+ if (configurationProperties == null) {
+ configurationProperties = Collections.emptyMap();
+ }
+
+ if ("cluster-env".equals(configType)) {
+ configurationProperties = new HashMap<String, String>(configurationProperties);
+ configurationProperties.put("security_enabled", (kerberosEnabled) ? "true" : "false");
+ }
+
+ properties.put("properties", configurationProperties);
+ requestConfigurations.put(configType, properties);
+ }
+ }
+
+ // Apply the current Kerberos properties...
+ for (Map.Entry<String, Map<String, String>> configuration : kerberosConfigurations.entrySet()) {
+ String configType = configuration.getKey();
+ Map<String, String> configurationProperties = configuration.getValue();
+
+ if ((configurationProperties != null) && !configurationProperties.isEmpty()) {
+ Map<String, Map<String, String>> requestConfiguration = requestConfigurations.get(configType);
+
+ if (requestConfiguration == null) {
+ requestConfiguration = new HashMap<String, Map<String, String>>();
+ requestConfigurations.put(configType, requestConfiguration);
+ }
+
+ Map<String, String> requestConfigurationProperties = requestConfiguration.get("properties");
+ if (requestConfigurationProperties == null) {
+ requestConfigurationProperties = new HashMap<String, String>();
+ } else {
+ requestConfigurationProperties = new HashMap<String, String>(requestConfigurationProperties);
+ }
+
+ requestConfigurationProperties.putAll(configurationProperties);
+ requestConfiguration.put("properties", requestConfigurationProperties);
+ }
+ }
+
+ StackAdvisorRequest request = StackAdvisorRequest.StackAdvisorRequestBuilder
+ .forStack(stackVersion.getStackName(), stackVersion.getStackVersion())
+ .forServices(new ArrayList<String>(services))
+ .forHosts(hostNames)
+ .withConfigurations(requestConfigurations)
+ .ofType(StackAdvisorRequest.StackAdvisorRequestType.CONFIGURATIONS)
+ .build();
+
+ try {
+ RecommendationResponse response = stackAdvisorHelper.recommend(request);
+
+ RecommendationResponse.Recommendation recommendation = (response == null) ? null : response.getRecommendations();
+ RecommendationResponse.Blueprint blueprint = (recommendation == null) ? null : recommendation.getBlueprint();
+ Map<String, RecommendationResponse.BlueprintConfigurations> configurations = (blueprint == null) ? null : blueprint.getConfigurations();
+
+ if (configurations != null) {
+ for (Map.Entry<String, RecommendationResponse.BlueprintConfigurations> configuration : configurations.entrySet()) {
+ String configType = configuration.getKey();
+ Map<String, String> recommendedConfigProperties = configuration.getValue().getProperties();
+ Map<String, String> existingConfigProperties = (existingConfigurations == null) ? null : existingConfigurations.get(configType);
+ Map<String, String> kerberosConfigProperties = kerberosConfigurations.get(configType);
+ Set<String> ignoreProperties = (propertiesToIgnore == null) ? null : propertiesToIgnore.get(configType);
+
+ for (Map.Entry<String, String> property : recommendedConfigProperties.entrySet()) {
+ String propertyName = property.getKey();
+
+ if ((ignoreProperties == null) || !ignoreProperties.contains(propertyName)) {
+ String recommendedValue = property.getValue();
+
+ if (kerberosConfigProperties == null) {
+ // There is no explicit update for this property from the Kerberos Descriptor...
+ // add the config and property if it also does not exist in the existing configurations
+ if ((existingConfigProperties == null) || !existingConfigProperties.containsKey(propertyName)) {
+ LOG.debug("Adding Kerberos configuration based on StackAdvisor recommendation:" +
+ "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
+ configType, propertyName, recommendedValue);
+
+ HashMap<String, String> properties = new HashMap<String, String>();
+ properties.put(propertyName, recommendedValue);
+ kerberosConfigurations.put(configType, properties);
+ }
+ } else {
+ String value = kerberosConfigProperties.get(propertyName);
+ if (value == null) {
+ // There is no explicit update for this property from the Kerberos Descriptor...
+ // add the property if it also does not exist in the existing configurations
+ if ((existingConfigProperties == null) || !existingConfigProperties.containsKey(propertyName)) {
+ LOG.debug("Adding Kerberos configuration based on StackAdvisor recommendation:" +
+ "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
+ configType, propertyName, recommendedValue);
+
+ kerberosConfigProperties.put(propertyName, recommendedValue);
+ }
+ } else if (!value.equals(recommendedValue)) {
+ // If the recommended value is a change, automatically change it.
+ LOG.debug("Updating Kerberos configuration based on StackAdvisor recommendation:" +
+ "\n\tConfigType: {}\n\tProperty: {}\n\tOld Value: {}\n\tNew Value: {}",
+ configType, propertyName, value, recommendedValue);
+
+ kerberosConfigProperties.put(propertyName, recommendedValue);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ throw new AmbariException(e.getMessage(), e);
+ }
+ }
+
return kerberosConfigurations;
}
@@ -1911,16 +2089,8 @@ public class KerberosHelperImpl implements KerberosHelper {
: !"false".equalsIgnoreCase(value);
}
- /**
- * Given a list of KerberosIdentityDescriptors, returns a Map fo configuration types to property
- * names and values.
- * <p/>
- * The property names and values are not expected to have any variable replacements done.
- *
- * @param identityDescriptors a List of KerberosIdentityDescriptor from which to retrieve configurations
- * @return a Map of configuration types to property name/value pairs (as a Map)
- */
- private Map<String, Map<String, String>> getConfigurations(List<KerberosIdentityDescriptor> identityDescriptors) {
+ @Override
+ public Map<String, Map<String, String>> getIdentityConfigurations(List<KerberosIdentityDescriptor> identityDescriptors) {
Map<String, Map<String, String>> map = new HashMap<String, Map<String, String>>();
if (identityDescriptors != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index dcedd51..9f42901 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -2459,11 +2459,6 @@ public class BlueprintConfigurationProcessor {
}
}
-
- if(clusterTopology.isClusterKerberosEnabled()) {
- configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, "security_enabled", "true");
- configTypesUpdated.add(CLUSTER_ENV_CONFIG_TYPE_NAME);
- }
}
/**
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
index 4c241d1..00b8a74 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
@@ -36,12 +36,14 @@ import org.apache.ambari.server.actionmanager.Request;
import org.apache.ambari.server.agent.CommandReport;
import org.apache.ambari.server.agent.ExecutionCommand;
import org.apache.ambari.server.controller.internal.CalculatedStatus;
+import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken;
import org.apache.ambari.server.utils.StageUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.inject.Inject;
import com.google.inject.Injector;
+import org.springframework.security.core.context.SecurityContextHolder;
/**
* Server Action Executor used to execute server-side actions (or tasks)
@@ -471,6 +473,12 @@ public class ServerActionExecutor {
try {
LOG.debug("Executing task #{}", taskId);
+ // Set an internal administrator user to be the authenticated user in the event an
+ // authorization check is needed while performing a server-side action.
+ InternalAuthenticationToken authentication = new InternalAuthenticationToken("server_action_executor");
+ authentication.setAuthenticated(true);
+ SecurityContextHolder.getContext().setAuthentication(authentication);
+
commandReport = execute(hostRoleCommand, executionCommand);
LOG.debug("Task #{} completed execution with status of {}",
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
index 359e651..3a945ad 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
@@ -36,9 +36,9 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Type;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -62,7 +62,8 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
protected void processServiceComponentHosts(Cluster cluster, KerberosDescriptor kerberosDescriptor, List<ServiceComponentHost> schToProcess,
Collection<String> identityFilter, String dataDirectory,
- Map<String, Map<String, String>> kerberosConfigurations) throws AmbariException {
+ Map<String, Map<String, String>> kerberosConfigurations,
+ boolean kerberosEnabled) throws AmbariException {
actionLog.writeStdOut("Processing Kerberos identities and configurations");
@@ -76,38 +77,35 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
// Create the file used to store details about principals and keytabs to create
File identityDataFile = new File(dataDirectory, KerberosIdentityDataFileWriter.DATA_FILE_NAME);
- // Group ServiceComponentHosts with their relevant hosts so we can create the relevant host-based
- // configurations once per host, rather than for every ServiceComponentHost we encounter
- Map<String, List<ServiceComponentHost>> hostServiceComponentHosts = new HashMap<String, List<ServiceComponentHost>>();
- for (ServiceComponentHost sch : schToProcess) {
- String hostName = sch.getHostName();
- List<ServiceComponentHost> serviceComponentHosts = hostServiceComponentHosts.get(hostName);
+ Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
+ KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter;
- if (serviceComponentHosts == null) {
- serviceComponentHosts = new ArrayList<ServiceComponentHost>();
- hostServiceComponentHosts.put(hostName, serviceComponentHosts);
- }
+ // Calculate the current host-specific configurations. These will be used to replace
+ // variables within the Kerberos descriptor data
+ Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptorProperties);
- serviceComponentHosts.add(sch);
+ actionLog.writeStdOut(String.format("Writing Kerberos identity data metadata file to %s", identityDataFile.getAbsolutePath()));
+ try {
+ kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
+ } catch (IOException e) {
+ String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
+ LOG.error(message, e);
+ actionLog.writeStdOut(message);
+ actionLog.writeStdErr(message + "\n" + e.getLocalizedMessage());
+ throw new AmbariException(message, e);
}
- Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
- KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter = null;
-
try {
- for (Map.Entry<String, List<ServiceComponentHost>> entry : hostServiceComponentHosts.entrySet()) {
- String hostName = entry.getKey();
- List<ServiceComponentHost> serviceComponentHosts = entry.getValue();
+ Set<String> services = new HashSet<String>();
+ Map<String, Set<String>> propertiesToIgnore = null;
- // Calculate the current host-specific configurations. These will be used to replace
- // variables within the Kerberos descriptor data
- Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, hostName, kerberosDescriptorProperties);
+ // Iterate over the components installed on the current host to get the service and
+ // component-level Kerberos descriptors in order to determine which principals,
+ // keytab files, and configurations need to be created or updated.
+ for (ServiceComponentHost sch : schToProcess) {
+ String hostName = sch.getHostName();
try {
- // Iterate over the components installed on the current host to get the service and
- // component-level Kerberos descriptors in order to determine which principals,
- // keytab files, and configurations need to be created or updated.
- for (ServiceComponentHost sch : serviceComponentHosts) {
String serviceName = sch.getServiceName();
String componentName = sch.getServiceComponentName();
@@ -116,15 +114,10 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
if (serviceDescriptor != null) {
List<KerberosIdentityDescriptor> serviceIdentities = serviceDescriptor.getIdentities(true);
- // Lazily create the KerberosIdentityDataFileWriter instance...
- if (kerberosIdentityDataFileWriter == null) {
- actionLog.writeStdOut(String.format("Writing Kerberos identity data metadata file to %s", identityDataFile.getAbsolutePath()));
- kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
- }
-
// Add service-level principals (and keytabs)
kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, serviceIdentities,
identityFilter, hostName, serviceName, componentName, kerberosConfigurations, configurations);
+ propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, propertiesToIgnore);
KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName);
@@ -139,9 +132,11 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
// Add component-level principals (and keytabs)
kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, componentIdentities,
identityFilter, hostName, serviceName, componentName, kerberosConfigurations, configurations);
+ propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
}
}
- }
+
+ services.add(serviceName);
} catch (IOException e) {
String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
LOG.error(message, e);
@@ -150,6 +145,9 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
throw new AmbariException(message, e);
}
}
+
+ kerberosHelper.applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations,
+ propertiesToIgnore, kerberosEnabled);
}
finally {
if (kerberosIdentityDataFileWriter != null) {
@@ -202,4 +200,30 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
return null;
}
}
+
+ private Map<String, Set<String>> gatherPropertiesToIgnore(List<KerberosIdentityDescriptor> identities,
+ Map<String, Set<String>> propertiesToIgnore) {
+ Map<String,Map<String,String>> identityConfigurations = kerberosHelper.getIdentityConfigurations(identities);
+ if ((identityConfigurations != null) && !identityConfigurations.isEmpty()) {
+ if(propertiesToIgnore == null) {
+ propertiesToIgnore = new HashMap<String, Set<String>>();
+ }
+
+ for (Map.Entry<String, Map<String, String>> entry : identityConfigurations.entrySet()) {
+ String configType = entry.getKey();
+ Map<String, String> properties = entry.getValue();
+
+ if ((properties != null) && !properties.isEmpty()) {
+ Set<String> propertyNames = propertiesToIgnore.get(configType);
+ if (propertyNames == null) {
+ propertyNames = new HashSet<String>();
+ propertiesToIgnore.put(configType, propertyNames);
+ }
+ propertyNames.addAll(properties.keySet());
+ }
+ }
+ }
+
+ return propertiesToIgnore;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
index 8ab04ff..f0a5440 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
@@ -112,7 +112,7 @@ public class PrepareDisableKerberosServerAction extends AbstractPrepareKerberosS
actionLog.writeStdOut(String.format("Processing %d components", schCount));
}
- processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations);
+ processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations, false);
actionLog.writeStdOut("Determining configuration changes");
// Ensure the cluster-env/security_enabled flag is set properly
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
index 84248dc..a7c3861 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
@@ -78,7 +78,7 @@ public class PrepareEnableKerberosServerAction extends PrepareKerberosIdentities
actionLog.writeStdOut(String.format("Processing %d components", schCount));
}
- processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations);
+ processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations, true);
processAuthToLocalRules(cluster, kerberosDescriptor, schToProcess, kerberosConfigurations, getDefaultRealm(commandParameters));
// Ensure the cluster-env/security_enabled flag is set properly
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
index fba3eea..8f8f67d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
@@ -35,7 +35,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ConcurrentMap;
/**
@@ -94,7 +93,7 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber
actionLog.writeStdOut(String.format("Processing %d components", schCount));
}
- processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations);
+ processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations, true);
if ("true".equalsIgnoreCase(getCommandParameterValue(commandParameters, UPDATE_CONFIGURATIONS))) {
processAuthToLocalRules(cluster, kerberosDescriptor, schToProcess, kerberosConfigurations, getDefaultRealm(commandParameters));
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 1e89272..0aae007 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -141,7 +141,20 @@ public class ClusterConfigurationRequest {
// apply Kerberos specific configurations
Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
.getServiceConfigurationUpdates(cluster, existingConfigurations,
- new HashSet<String>(blueprint.getServices()), false);
+ new HashSet<String>(blueprint.getServices()), false, true, false);
+
+ // ******************************************************************************************
+ // Since Kerberos is being enabled, make sure the cluster-env/security_enabled property is
+ // set to "true"
+ Map<String, String> clusterEnv = updatedConfigs.get("cluster-env");
+
+ if(clusterEnv == null) {
+ clusterEnv = new HashMap<String,String>();
+ updatedConfigs.put("cluster-env", clusterEnv);
+ }
+
+ clusterEnv.put("security_enabled", "true");
+ // ******************************************************************************************
for (String configType : updatedConfigs.keySet()) {
Map<String, String> propertyMap = updatedConfigs.get(configType);
@@ -197,7 +210,7 @@ public class ClusterConfigurationRequest {
// apply Kerberos specific configurations
Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
.getServiceConfigurationUpdates(cluster, existingConfigurations,
- new HashSet<String>(blueprint.getServices()), false);
+ new HashSet<String>(blueprint.getServices()), false, true, false);
// retrieve hostgroup for component names extracted from variables like "{clusterHostInfo.(component_name)
// _host}"
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index ca7d2dd..da2fc47 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -341,7 +341,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
ranger_admin_host = ranger_admin_hosts[0]
policymgr_external_url = "%s://%s:%s" % (protocol, ranger_admin_host, port)
- putRangerAdminProperty('policymgr_external_url', policymgr_external_url)
+ putRangerAdminProperty('policymgr_external_url', policymgr_external_url)
rangerServiceVersion = [service['StackServices']['service_version'] for service in services["services"] if service['StackServices']['service_name'] == 'RANGER'][0]
if rangerServiceVersion == '0.4.0':
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
index d5531a6..60a8dde 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
@@ -76,6 +76,7 @@ public class StackAdvisorBlueprintProcessorTest {
expect(clusterTopology.getHostGroupInfo()).andReturn(createHostGroupInfo()).anyTimes();
expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
+ expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
expect(blueprint.getStack()).andReturn(stack).anyTimes();
expect(stack.getVersion()).andReturn("2.3").anyTimes();
expect(stack.getName()).andReturn("HDP").anyTimes();
@@ -104,6 +105,7 @@ public class StackAdvisorBlueprintProcessorTest {
expect(clusterTopology.getHostGroupInfo()).andReturn(createHostGroupInfo()).anyTimes();
expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
+ expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
expect(blueprint.getStack()).andReturn(stack).anyTimes();
expect(stack.getVersion()).andReturn("2.3").anyTimes();
expect(stack.getName()).andReturn("HDP").anyTimes();
@@ -131,6 +133,7 @@ public class StackAdvisorBlueprintProcessorTest {
expect(clusterTopology.getHostGroupInfo()).andReturn(createHostGroupInfo()).anyTimes();
expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
+ expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
expect(blueprint.getStack()).andReturn(stack).anyTimes();
expect(stack.getVersion()).andReturn("2.3").anyTimes();
expect(stack.getName()).andReturn("HDP").anyTimes();
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
index 8861549..869d187 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
@@ -30,6 +30,9 @@ import org.apache.ambari.server.actionmanager.RequestFactory;
import org.apache.ambari.server.actionmanager.Stage;
import org.apache.ambari.server.actionmanager.StageFactory;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
import org.apache.ambari.server.controller.internal.RequestStageContainer;
import org.apache.ambari.server.controller.spi.ClusterController;
@@ -211,6 +214,7 @@ public class KerberosHelperTest extends EasyMockSupport {
bind(CredentialStoreService.class).to(CredentialStoreServiceImpl.class);
bind(CreatePrincipalsServerAction.class).toInstance(createMock(CreatePrincipalsServerAction.class));
bind(CreateKeytabFilesServerAction.class).toInstance(createMock(CreateKeytabFilesServerAction.class));
+ bind(StackAdvisorHelper.class).toInstance(createMock(StackAdvisorHelper.class));
}
});
@@ -2168,6 +2172,37 @@ public class KerberosHelperTest extends EasyMockSupport {
expect(metaInfo.getKerberosDescriptor("HDP", "2.2")).andReturn(kerberosDescriptor).atLeastOnce();
expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).atLeastOnce();
+ RecommendationResponse.BlueprintConfigurations coreSiteRecommendation = createMock(RecommendationResponse.BlueprintConfigurations.class);
+ expect(coreSiteRecommendation.getProperties()).andReturn(Collections.singletonMap("newPropertyRecommendation", "newPropertyRecommendation"));
+
+ RecommendationResponse.BlueprintConfigurations newTypeRecommendation = createMock(RecommendationResponse.BlueprintConfigurations.class);
+ expect(newTypeRecommendation.getProperties()).andReturn(Collections.singletonMap("newTypeRecommendation", "newTypeRecommendation"));
+
+ RecommendationResponse.BlueprintConfigurations type1Recommendation = createMock(RecommendationResponse.BlueprintConfigurations.class);
+ expect(type1Recommendation.getProperties()).andReturn(Collections.singletonMap("replacement1", "not replaced"));
+
+ RecommendationResponse.BlueprintConfigurations service1SiteRecommendation = createMock(RecommendationResponse.BlueprintConfigurations.class);
+ expect(service1SiteRecommendation.getProperties()).andReturn(Collections.singletonMap("component1b.property", "replaced value"));
+
+ Map<String, RecommendationResponse.BlueprintConfigurations> configurations = new HashMap<String, RecommendationResponse.BlueprintConfigurations>();
+ configurations.put("core-site", coreSiteRecommendation);
+ configurations.put("new-type", newTypeRecommendation);
+ configurations.put("type1", type1Recommendation);
+ configurations.put("service1-site", service1SiteRecommendation);
+
+ RecommendationResponse.Blueprint blueprint = createMock(RecommendationResponse.Blueprint.class);
+ expect(blueprint.getConfigurations()).andReturn(configurations).once();
+
+ RecommendationResponse.Recommendation recommendations = createMock(RecommendationResponse.Recommendation.class);
+ expect(recommendations.getBlueprint()).andReturn(blueprint).once();
+
+ RecommendationResponse recommendationResponse = createMock(RecommendationResponse.class);
+ expect(recommendationResponse.getRecommendations()).andReturn(recommendations).once();
+
+ StackAdvisorHelper stackAdvisorHelper = injector.getInstance(StackAdvisorHelper.class);
+ expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(null).once();
+ expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(recommendationResponse).once();
+
final Service service1 = createMockService("SERVICE1",
new HashMap<String, ServiceComponent>() {
{
@@ -2267,10 +2302,10 @@ public class KerberosHelperTest extends EasyMockSupport {
injector.getInstance(AmbariMetaInfo.class).init();
Map<String, Map<String, String>> updates1 = kerberosHelper.getServiceConfigurationUpdates(
- cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE2", "SERVICE3")), false);
+ cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE2", "SERVICE3")), false, true, true);
Map<String, Map<String, String>> updates2 = kerberosHelper.getServiceConfigurationUpdates(
- cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE3")), false);
+ cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE3")), false, true, true);
verifyAll();
@@ -2322,6 +2357,13 @@ public class KerberosHelperTest extends EasyMockSupport {
assertEquals(expectedUpdates, updates1);
expectedUpdates.remove("service2-site");
+ expectedUpdates.get("core-site").put("newPropertyRecommendation", "newPropertyRecommendation");
+ expectedUpdates.get("service1-site").put("component1b.property", "replaced value");
+ expectedUpdates.put("new-type", new HashMap<String, String>() {
+ {
+ put("newTypeRecommendation", "newTypeRecommendation");
+ }
+ });
assertEquals(expectedUpdates, updates2);
http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
index d1886b9..7309d5b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
@@ -48,6 +48,7 @@ import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.anyString;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.eq;
import static org.easymock.EasyMock.verify;
/**
@@ -145,7 +146,7 @@ public class ClusterConfigurationRequestTest {
expect(kerberosHelper.ensureHeadlessIdentities(anyObject(Cluster.class), anyObject(Map.class), anyObject
(Set.class))).andReturn(true).once();
expect(kerberosHelper.getServiceConfigurationUpdates(anyObject(Cluster.class), anyObject(Map.class), anyObject
- (Set.class), anyBoolean())).andReturn(kerberosConfig).once();
+ (Set.class), anyBoolean(), anyBoolean(), eq(false))).andReturn(kerberosConfig).once();
PowerMock.replay(stack, blueprint, topology, controller, clusters, kerberosHelper, ambariContext,
[11/19] ambari git commit: AMBARI-14676. Create DB model for admin
setting (Ajit Kumar via smohanty)
Posted by nc...@apache.org.
AMBARI-14676. Create DB model for admin setting (Ajit Kumar via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/58903d89
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/58903d89
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/58903d89
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 58903d89cf0cd50507b6cfddc25c405d6f725c5c
Parents: cbed640
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Jan 18 20:13:37 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Jan 18 20:13:37 2016 -0800
----------------------------------------------------------------------
.../src/main/resources/Ambari-DDL-Derby-CREATE.sql | 12 ++++++++++++
.../src/main/resources/Ambari-DDL-MySQL-CREATE.sql | 11 +++++++++++
.../src/main/resources/Ambari-DDL-Oracle-CREATE.sql | 11 +++++++++++
.../src/main/resources/Ambari-DDL-Postgres-CREATE.sql | 12 ++++++++++++
.../resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql | 13 +++++++++++++
.../main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 10 ++++++++++
.../src/main/resources/Ambari-DDL-SQLServer-CREATE.sql | 11 +++++++++++
7 files changed, 80 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
index 72413ce..9fc0a16 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
@@ -655,6 +655,16 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+CREATE TABLE adminsetting (
+ id BIGINT NOT NULL,
+ name VARCHAR(255) NOT NULL UNIQUE,
+ setting_type VARCHAR(255) NOT NULL,
+ content VARCHAR(3000) NOT NULL,
+ updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+ update_timestamp BIGINT NOT NULL,
+ PRIMARY KEY (id)
+);
+
-- tasks indices --
CREATE INDEX idx_stage_request_id ON stage (request_id);
CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -1023,6 +1033,8 @@ INSERT INTO ambari_sequences (sequence_name, sequence_value)
union all
select 'topology_host_group_id_seq', 0 FROM SYSIBM.SYSDUMMY1
union all
+ select 'admin_setting_id_seq', 0 FROM SYSIBM.SYSDUMMY1
+ union all
select 'hostcomponentstate_id_seq', 0 FROM SYSIBM.SYSDUMMY1;
INSERT INTO adminresourcetype (resource_type_id, resource_type_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
index 280b898..775eef2 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
@@ -662,6 +662,16 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+CREATE TABLE adminsetting (
+ id BIGINT NOT NULL,
+ name VARCHAR(255) NOT NULL UNIQUE,
+ setting_type VARCHAR(255) NOT NULL,
+ content TEXT NOT NULL,
+ updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+ update_timestamp BIGINT NOT NULL,
+ PRIMARY KEY (id)
+);
+
-- tasks indices --
CREATE INDEX idx_stage_request_id ON stage (request_id);
CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -987,6 +997,7 @@ INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_lo
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_logical_task_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_request_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_host_group_id_seq', 0);
+INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('admin_setting_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('hostcomponentstate_id_seq', 0);
insert into adminresourcetype (resource_type_id, resource_type_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
index b2ba909..ea9e85f 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
@@ -651,6 +651,16 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+CREATE TABLE adminsetting (
+ id NUMBER(19) NOT NULL,
+ name VARCHAR(255) NOT NULL UNIQUE,
+ setting_type VARCHAR(255) NOT NULL,
+ content CLOB NOT NULL,
+ updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+ update_timestamp NUMBER(19) NOT NULL,
+ PRIMARY KEY (id)
+);
+
-- tasks indices --
CREATE INDEX idx_stage_request_id ON stage (request_id);
CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -977,6 +987,7 @@ INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_lo
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_logical_task_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_request_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_host_group_id_seq', 0);
+INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('admin_setting_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('hostcomponentstate_id_seq', 0);
INSERT INTO metainfo("metainfo_key", "metainfo_value") values ('version', '${ambariVersion}');
http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
index cec88d3..bb49583 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
@@ -655,6 +655,16 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+CREATE TABLE adminsetting (
+ id BIGINT NOT NULL,
+ name VARCHAR(255) NOT NULL UNIQUE,
+ setting_type VARCHAR(255) NOT NULL,
+ content TEXT NOT NULL,
+ updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+ update_timestamp BIGINT NOT NULL,
+ PRIMARY KEY (id)
+);
+
-- tasks indices --
CREATE INDEX idx_stage_request_id ON stage (request_id);
CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -1023,6 +1033,8 @@ INSERT INTO ambari_sequences (sequence_name, sequence_value)
union all
select 'topology_host_group_id_seq', 0
union all
+ select 'admin_setting_id_seq', 0
+ union all
select 'hostcomponentstate_id_seq', 0;
INSERT INTO adminresourcetype (resource_type_id, resource_type_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
index 99248bc..149c9cf 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
@@ -736,6 +736,17 @@ CREATE TABLE ambari.topology_logical_task (
);
GRANT ALL PRIVILEGES ON TABLE ambari.topology_logical_task TO :username;
+CREATE TABLE ambari.adminsetting (
+ id BIGINT NOT NULL,
+ name VARCHAR(255) NOT NULL UNIQUE,
+ setting_type VARCHAR(255) NOT NULL,
+ content TEXT NOT NULL,
+ updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+ update_timestamp BIGINT NOT NULL,
+ PRIMARY KEY (id)
+);
+GRANT ALL PRIVILEGES ON TABLE ambari.adminsetting TO :username;
+
-- tasks indices --
CREATE INDEX idx_stage_request_id ON ambari.stage (request_id);
CREATE INDEX idx_hrc_request_id ON ambari.host_role_command (request_id);
@@ -1121,6 +1132,8 @@ INSERT INTO ambari.ambari_sequences (sequence_name, sequence_value)
union all
select 'topology_host_group_id_seq', 0
union all
+ select 'admin_setting_id_seq', 0
+ union all
select 'hostcomponentstate_id_seq', 0;
INSERT INTO ambari.adminresourcetype (resource_type_id, resource_type_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
index 7831836..7d88568 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
@@ -652,6 +652,15 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY (id)
);
+CREATE TABLE adminsetting (
+ id NUMERIC(19) NOT NULL,
+ name VARCHAR(255) NOT NULL UNIQUE,
+ setting_type VARCHAR(255) NOT NULL,
+ content TEXT NOT NULL,
+ updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+ update_timestamp NUMERIC(19) NOT NULL,
+ PRIMARY KEY (id)
+);
-- tasks indices --
CREATE INDEX idx_stage_request_id ON stage (request_id);
CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -975,6 +984,7 @@ INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_lo
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_logical_task_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_request_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_host_group_id_seq', 0);
+INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('admin_setting_id_seq', 0);
INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('hostcomponentstate_id_seq', 0);
insert into adminresourcetype (resource_type_id, resource_type_name)
http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
index 7f9ea12..4374fc6 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
@@ -760,6 +760,16 @@ CREATE TABLE topology_logical_task (
PRIMARY KEY CLUSTERED (id)
);
+CREATE TABLE adminsetting (
+ id BIGINT NOT NULL,
+ name VARCHAR(255) NOT NULL UNIQUE,
+ setting_type VARCHAR(255) NOT NULL,
+ content TEXT NOT NULL,
+ updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+ update_timestamp BIGINT NOT NULL,
+ PRIMARY KEY (id)
+);
+
-- tasks indices --
CREATE INDEX idx_stage_request_id ON stage (request_id);
CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -1090,6 +1100,7 @@ BEGIN TRANSACTION
('topology_logical_task_id_seq', 0),
('topology_request_id_seq', 0),
('topology_host_group_id_seq', 0),
+ ('admin_setting_id_seq', 0),
('hostcomponentstate_id_seq', 0);
insert into adminresourcetype (resource_type_id, resource_type_name)
[03/19] ambari git commit: AMBARI-14704 : Restart storm fails with a
metrics storm sink jar related error sometimes - Commit 2. (avijayan)
Posted by nc...@apache.org.
AMBARI-14704 : Restart storm fails with a metrics storm sink jar related error sometimes - Commit 2. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8684e450
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8684e450
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8684e450
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8684e45065b3849b9ba86588309f75c535cde39b
Parents: ba3db6d
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Mon Jan 18 09:22:11 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Mon Jan 18 09:22:15 2016 -0800
----------------------------------------------------------------------
.../STORM/0.9.1.2.1/package/scripts/storm.py | 11 -----------
1 file changed, 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8684e450/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
index 98abb41..43736fa 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
@@ -87,17 +87,6 @@ def storm(name=None):
group=params.user_group
)
- # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2
- Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"),
- action="delete")
- # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions
- Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete")
-
- Execute(format("{sudo} ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
- not_if=format("ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
- only_if=format("ls {metric_collector_sink_jar}")
- )
-
File(format("{conf_dir}/storm-env.sh"),
owner=params.storm_user,
content=InlineTemplate(params.storm_env_sh_template)
[12/19] ambari git commit: AMBARI-14710. Add retry logic for REST
calls between Ambari and Ranger(gautam)
Posted by nc...@apache.org.
AMBARI-14710. Add retry logic for REST calls between Ambari and Ranger(gautam)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/869e0858
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/869e0858
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/869e0858
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 869e0858ccd835baa53ec3b88f50439cdecfd2c7
Parents: 58903d8
Author: Gautam Borad <ga...@apache.org>
Authored: Mon Jan 18 22:24:43 2016 +0530
Committer: Gautam Borad <ga...@apache.org>
Committed: Tue Jan 19 09:56:21 2016 +0530
----------------------------------------------------------------------
.../libraries/functions/ranger_functions.py | 8 +++-----
.../libraries/functions/ranger_functions_v2.py | 8 +++-----
2 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/869e0858/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
index 543a31f..c59dc8d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
@@ -219,13 +219,11 @@ class Rangeradmin:
return response_code
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
- Logger.error("Connection to Ranger Admin failed. Http status code - {0}. \n {1}".format(e.code, e.read()))
+ raise Fail("Connection to Ranger Admin failed. Http status code - {0}. \n {1}".format(e.code, e.read()))
else:
- Logger.error("Connection to Ranger Admin failed. Reason - {0}.".format(e.reason))
- return None
+ raise Fail("Connection to Ranger Admin failed. Reason - {0}.".format(e.reason))
except httplib.BadStatusLine, e:
- Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
- return None
+ raise Fail("Ranger Admin service is not reachable, please restart the service and then try again")
except TimeoutError:
raise Fail("Connection to Ranger Admin failed. Reason - timeout")
http://git-wip-us.apache.org/repos/asf/ambari/blob/869e0858/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
index 80965a6..a72be33 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
@@ -183,13 +183,11 @@ class RangeradminV2:
return response_code
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError):
- Logger.error("Connection failed to Ranger Admin. Http status code - {0}. \n {1}".format(e.code, e.read()))
+ raise Fail("Connection failed to Ranger Admin. Http status code - {0}. \n {1}".format(e.code, e.read()))
else:
- Logger.error("Connection failed to Ranger Admin. Reason - {0}.".format(e.reason))
- return None
+ raise Fail("Connection failed to Ranger Admin. Reason - {0}.".format(e.reason))
except httplib.BadStatusLine, e:
- Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
- return None
+ raise Fail("Ranger Admin service is not reachable, please restart the service and then try again")
except TimeoutError:
raise Fail("Connection failed to Ranger Admin. Reason - timeout")
[18/19] ambari git commit: AMBARI-14725. HDP-2.3 to HDP-2.4 Upgrade
Pack is missing Mahout and Phoenix Query Server (ncole)
Posted by nc...@apache.org.
AMBARI-14725. HDP-2.3 to HDP-2.4 Upgrade Pack is missing Mahout and Phoenix Query Server (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/09d3e8bd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/09d3e8bd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/09d3e8bd
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 09d3e8bd636b8a4cf150d3b6b62c9738ee46bc2b
Parents: 07b8a3a
Author: Nate Cole <nc...@hortonworks.com>
Authored: Tue Jan 19 12:48:38 2016 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Tue Jan 19 12:48:38 2016 -0500
----------------------------------------------------------------------
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/09d3e8bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index c15769b..e579ee0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -164,6 +164,7 @@
<service name="HBASE">
<component>HBASE_REGIONSERVER</component>
+ <component>PHOENIX_QUERY_SERVER</component>
</service>
<service name="YARN">
@@ -284,6 +285,10 @@
<component>SQOOP</component>
</service>
+ <service name="MAHOUT">
+ <component>MAHOUT</component>
+ </service>
+
<service name="HIVE">
<component>HIVE_CLIENT</component>
<component>HCAT</component>
@@ -530,6 +535,12 @@
<task xsi:type="restart-task" />
</upgrade>
</component>
+
+ <component name="PHOENIX_QUERY_SERVER">
+ <upgrade>
+ <task xsi:type="restart-task"/>
+ </upgrade>
+ </component>
</service>
<service name="TEZ">
@@ -551,6 +562,14 @@
</component>
</service>
+ <service name="MAHOUT">
+ <component name="MAHOUT">
+ <upgrade>
+ <task xsi:type="restart-task" />
+ </upgrade>
+ </component>
+ </service>
+
<service name="SQOOP">
<component name="SQOOP">
<upgrade>
[17/19] ambari git commit: AMBARI-14719 After enabling NN HA AMS
shows critical alert. (ababiichuk)
Posted by nc...@apache.org.
AMBARI-14719 After enabling NN HA AMS shows critical alert. (ababiichuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/07b8a3ab
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/07b8a3ab
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/07b8a3ab
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 07b8a3ab2c8fa72a9f8d63fa5564a1f13835e26c
Parents: 8f6b45d
Author: ababiichuk <ab...@hortonworks.com>
Authored: Tue Jan 19 15:54:51 2016 +0200
Committer: ababiichuk <ab...@hortonworks.com>
Committed: Tue Jan 19 17:20:25 2016 +0200
----------------------------------------------------------------------
.../configs/config_recommendation_parser.js | 9 +++++--
...onfig_with_override_recommendation_parser.js | 2 +-
.../mixins/common/configs/enhanced_configs.js | 13 ++--------
ambari-web/app/views/common/controls_view.js | 7 ++---
.../config_recommendation_parser_test.js | 27 +++++++++++++++++++-
5 files changed, 38 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/app/mixins/common/configs/config_recommendation_parser.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/config_recommendation_parser.js b/ambari-web/app/mixins/common/configs/config_recommendation_parser.js
index b5e2f2d..826ce19 100644
--- a/ambari-web/app/mixins/common/configs/config_recommendation_parser.js
+++ b/ambari-web/app/mixins/common/configs/config_recommendation_parser.js
@@ -290,9 +290,14 @@ App.ConfigRecommendationParser = Em.Mixin.create(App.ConfigRecommendations, {
* @param parentProperties
* @param name
* @param fileName
+ * @param [configGroup]
* @returns {boolean}
*/
- allowUpdateProperty: function (parentProperties, name, fileName) {
- return true;
+ allowUpdateProperty: function (parentProperties, name, fileName, configGroup) {
+ try {
+ return Em.get(this.getRecommendation(name, fileName, configGroup), 'saveRecommended');
+ } catch (e) {
+ return true;
+ }
}
});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js b/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js
index 7002110..ad06cde 100644
--- a/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js
+++ b/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js
@@ -46,7 +46,7 @@ App.ConfigWithOverrideRecommendationParser = Em.Mixin.create(App.ConfigRecommend
* @protected
*/
_updateOverride: function(config, recommendedValue, parentProperties, configGroup) {
- var updateValue = this.allowUpdateProperty(parentProperties, Em.get(config, 'name'), Em.get(config, 'filename'));
+ var updateValue = this.allowUpdateProperty(parentProperties, Em.get(config, 'name'), Em.get(config, 'filename'), configGroup.get('name'));
var override = config.getOverride(configGroup.get('name'));
if (override) {
this._updateConfigByRecommendation(override, recommendedValue, parentProperties);
http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/app/mixins/common/configs/enhanced_configs.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/enhanced_configs.js b/ambari-web/app/mixins/common/configs/enhanced_configs.js
index 85d157e..303b0c7 100644
--- a/ambari-web/app/mixins/common/configs/enhanced_configs.js
+++ b/ambari-web/app/mixins/common/configs/enhanced_configs.js
@@ -146,7 +146,7 @@ App.EnhancedConfigsMixin = Em.Mixin.create(App.ConfigWithOverrideRecommendationP
/**
* sends request to get values for dependent configs
* @param {{type: string, name: string}[]} changedConfigs - list of changed configs to track recommendations
- * @param {Function} onComplete
+ * @param {Function} [onComplete]
* @returns {$.ajax|null}
*/
loadConfigRecommendations: function(changedConfigs, onComplete) {
@@ -379,15 +379,6 @@ App.EnhancedConfigsMixin = Em.Mixin.create(App.ConfigWithOverrideRecommendationP
*/
removeCurrentFromDependentList: function (config, saveRecommended) {
var recommendation = this.getRecommendation(config.get('name'), config.get('filename'), config.get('group.name'));
- if (recommendation) {
- try {
- if (this.saveRecommendation(recommendation)) {
- this.undoRedoRecommended([recommendation], saveRecommended);
- this.set('recommendationTimeStamp', (new Date).getTime());
- }
- } catch(e) {
- console.warn(e.message);
- }
- }
+ if (recommendation) this.saveRecommendation(recommendation, saveRecommended);
}
});
http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/app/views/common/controls_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/controls_view.js b/ambari-web/app/views/common/controls_view.js
index 48493f0..c9281a8 100644
--- a/ambari-web/app/views/common/controls_view.js
+++ b/ambari-web/app/views/common/controls_view.js
@@ -103,6 +103,7 @@ App.SupportsDependentConfigs = Ember.Mixin.create({
var saveRecommended = (config.get('value') === config.get('recommendedValue'));
var type = App.config.getConfigTagFromFileName(config.get('filename'));
var p = App.configsCollection.getConfig(App.config.configId(name, type));
+ controller.removeCurrentFromDependentList(config, saveRecommended);
if ((p && Em.get(p, 'propertyDependedBy.length') > 0 || Em.get(p, 'displayType') === 'user') && config.get('oldValue') !== config.get('value')) {
var old = config.get('oldValue');
config.set('oldValue', config.get('value'));
@@ -110,11 +111,7 @@ App.SupportsDependentConfigs = Ember.Mixin.create({
"type": type,
"name": name,
"old_value": Em.isNone(old) ? config.get('initialValue') : old
- }], function() {
- controller.removeCurrentFromDependentList(config, saveRecommended);
- });
- } else {
- controller.removeCurrentFromDependentList(config, saveRecommended);
+ }]);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js b/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js
index e895859..35f84cd 100644
--- a/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js
+++ b/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js
@@ -71,6 +71,7 @@ describe('App.ConfigRecommendationParser', function() {
sinon.stub(App.configsCollection, 'getConfigByName', function(name, fileName) {
return { name: name, filename: fileName };
});
+ sinon.stub(instanceObject, 'allowUpdateProperty').returns(true);
sinon.spy(instanceObject, 'updateCallback');
sinon.spy(instanceObject, 'removeCallback');
@@ -83,6 +84,7 @@ describe('App.ConfigRecommendationParser', function() {
afterEach(function() {
App.configsCollection.getConfigByName.restore();
+ instanceObject.allowUpdateProperty.restore();
instanceObject.updateCallback.restore();
instanceObject.removeCallback.restore();
instanceObject.updateBoundariesCallback.restore();
@@ -449,9 +451,32 @@ describe('App.ConfigRecommendationParser', function() {
});
describe('#allowUpdateProperty', function() {
+
+
+
+ var cases = [{
+ saveRecommended: true
+ },{
+ saveRecommended: false
+ }];
+
+ cases.forEach(function(c) {
+ describe('allowUpdateProperty based on saveRecommended:' + c.saveRecommended, function() {
+ beforeEach(function() {
+ sinon.stub(instanceObject, 'getRecommendation').returns({saveRecommended: c.saveRecommended});
+ });
+ afterEach(function() {
+ instanceObject.getRecommendation.restore();
+ });
+ it('default value for allowUpdateProperty is true', function() {
+ expect(instanceObject.allowUpdateProperty()).to.equal(c.saveRecommended);
+ });
+ });
+ });
+
it('default value for allowUpdateProperty is true', function() {
expect(instanceObject.allowUpdateProperty()).to.be.true;
- })
+ });
});
});
[02/19] ambari git commit: AMBARI-14705. nodes in non default config
group do no get prompted for restart to update stale configs (smohanty)
Posted by nc...@apache.org.
AMBARI-14705. nodes in non default config group do no get prompted for restart to update stale configs (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba3db6d7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba3db6d7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba3db6d7
Branch: refs/heads/branch-dev-patch-upgrade
Commit: ba3db6d79b7204b6fcf3e105d7aa4a472ca451b0
Parents: 40aa612
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Jan 18 08:32:14 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Jan 18 08:39:13 2016 -0800
----------------------------------------------------------------------
.../ambari/server/state/ConfigHelper.java | 7 ++-
.../server/state/cluster/ClusterImpl.java | 11 ++++-
.../ambari/server/state/ConfigHelperTest.java | 52 +++++++++++++++++---
3 files changed, 62 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3db6d7/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 83d3a3a..d311017 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -379,6 +379,10 @@ public class ConfigHelper {
if (stale == null) {
stale = calculateIsStaleConfigs(sch);
staleConfigsCache.put(sch, stale);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Config staleness for " +
+ sch.getServiceComponentName() + " on host " + sch.getHostName() + " - " + stale);
+ }
}
return stale;
}
@@ -1134,10 +1138,11 @@ public class ConfigHelper {
* @return true if the tags are different in any way, even if not-specified
*/
private boolean isTagChanged(Map<String, String> desiredTags, Map<String, String> actualTags, boolean groupSpecificConfigs) {
- if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG)) && !groupSpecificConfigs) {
+ if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG))) {
return true;
}
+ // cluster level configs are already compared for staleness, now they match
// if the host has group specific configs for type we should ignore the cluster level configs and compare specifics
if (groupSpecificConfigs) {
actualTags.remove(CLUSTER_DEFAULT_TAG);
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3db6d7/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 14223fe..1d20322 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -487,9 +487,18 @@ public class ClusterImpl implements Cluster {
loadConfigGroups();
clusterGlobalLock.writeLock().lock();
try {
+ String hostList = "";
+ if(LOG.isDebugEnabled()) {
+ if (configGroup.getHosts() != null) {
+ for (Host host : configGroup.getHosts().values()) {
+ hostList += host.getHostName() + ", ";
+ }
+ }
+ }
+
LOG.debug("Adding a new Config group" + ", clusterName = "
+ getClusterName() + ", groupName = " + configGroup.getName()
- + ", tag = " + configGroup.getTag());
+ + ", tag = " + configGroup.getTag() + " with hosts " + hostList);
if (clusterConfigGroups.containsKey(configGroup.getId())) {
// The loadConfigGroups will load all groups to memory
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3db6d7/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index bf023c2..98424b7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -62,6 +62,8 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
@@ -74,6 +76,7 @@ import org.springframework.security.core.context.SecurityContextHolder;
@RunWith(Enclosed.class)
public class ConfigHelperTest {
public static class RunWithInMemoryDefaultTestModule {
+ private final static Logger LOG = LoggerFactory.getLogger(ConfigHelperTest.class);
private Clusters clusters;
private Injector injector;
private String clusterName;
@@ -220,6 +223,8 @@ public class ConfigHelperTest {
ConfigGroup configGroup = configGroupFactory.createNew(cluster, name,
tag, "", configMap, hostMap);
+ LOG.info("Config group created with tag " + tag);
+ configGroup.setTag(tag);
configGroup.persist();
cluster.addConfigGroup(configGroup);
@@ -660,17 +665,19 @@ public class ConfigHelperTest {
// Put a different version to check for change
hc.setDefaultVersionTag("version2");
schReturn.put("flume-conf", hc);
+
// set up mocks
ServiceComponentHost sch = createNiceMock(ServiceComponentHost.class);
// set up expectations
- expect(sch.getActualConfigs()).andReturn(schReturn).times(3);
- expect(sch.getHostName()).andReturn("h1").times(6);
- expect(sch.getClusterId()).andReturn(1l).times(3);
- expect(sch.getServiceName()).andReturn("FLUME").times(3);
- expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").times(3);
+ expect(sch.getActualConfigs()).andReturn(schReturn).times(6);
+ expect(sch.getHostName()).andReturn("h1").anyTimes();
+ expect(sch.getClusterId()).andReturn(1l).anyTimes();
+ expect(sch.getServiceName()).andReturn("FLUME").anyTimes();
+ expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").anyTimes();
replay(sch);
// Cluster level config changes
Assert.assertTrue(configHelper.isStaleConfigs(sch));
+
HostConfig hc2 = new HostConfig();
hc2.setDefaultVersionTag("version1");
schReturn.put("flume-conf", hc2);
@@ -678,13 +685,46 @@ public class ConfigHelperTest {
configHelper.invalidateStaleConfigsCache();
// Cluster level same configs
Assert.assertFalse(configHelper.isStaleConfigs(sch));
+
// Cluster level same configs but group specific configs for host have been updated
List<String> hosts = new ArrayList<String>();
hosts.add("h1");
List<Config> configs = new ArrayList<Config>();
- configs.add(new ConfigImpl("flume-conf"));
+ ConfigImpl configImpl = new ConfigImpl("flume-conf");
+ configImpl.setTag("FLUME1");
+ configs.add(configImpl);
addConfigGroup("configGroup1", "FLUME", hosts, configs);
+
+ // config group added for host - expect staleness
Assert.assertTrue(configHelper.isStaleConfigs(sch));
+
+ HostConfig hc3 = new HostConfig();
+ hc3.setDefaultVersionTag("version1");
+ hc3.getConfigGroupOverrides().put(1l, "FLUME1");
+ schReturn.put("flume-conf", hc3);
+ configHelper.invalidateStaleConfigsCache();
+
+ // version1 and FLUME1 - stale=false
+ Assert.assertFalse(configHelper.isStaleConfigs(sch));
+
+ HostConfig hc4 = new HostConfig();
+ hc4.setDefaultVersionTag("version1");
+ hc4.getConfigGroupOverrides().put(1l, "FLUME2");
+ schReturn.put("flume-conf", hc4);
+ configHelper.invalidateStaleConfigsCache();
+
+ // version1 and FLUME2 - stale=true
+ Assert.assertTrue(configHelper.isStaleConfigs(sch));
+
+ HostConfig hc5 = new HostConfig();
+ hc5.setDefaultVersionTag("version3");
+ hc5.getConfigGroupOverrides().put(1l, "FLUME1");
+ schReturn.put("flume-conf", hc5);
+ configHelper.invalidateStaleConfigsCache();
+
+ // version3 and FLUME1 - stale=true
+ Assert.assertTrue(configHelper.isStaleConfigs(sch));
+
verify(sch);
}
}
[08/19] ambari git commit: AMBARI-14656: dfs.allow.truncate property
check for Add Service wizard of HAWQ
Posted by nc...@apache.org.
AMBARI-14656: dfs.allow.truncate property check for Add Service wizard of HAWQ
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/132c5350
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/132c5350
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/132c5350
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 132c5350e40be4964b88d6f3feca275457523d92
Parents: 21b6b31
Author: Jun Aoki <ja...@apache.org>
Authored: Mon Jan 18 13:10:51 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Mon Jan 18 13:10:51 2016 -0800
----------------------------------------------------------------------
.../stacks/HDP/2.3/services/stack_advisor.py | 8 +++
.../stacks/2.3/common/test_stack_advisor.py | 60 ++++++++++++++++++++
2 files changed, 68 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/132c5350/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index ed241ff..0544f5a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -703,6 +703,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
validationItems.append({"config-name": 'dfs.namenode.inode.attributes.provider.class',
"item": self.getWarnItem(
"dfs.namenode.inode.attributes.provider.class needs to be set to 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer' if Ranger HDFS Plugin is enabled.")})
+
+ # Check if dfs.allow.truncate is true
+ if "HAWQ" in servicesList and \
+ not ("dfs.allow.truncate" in services["configurations"]["hdfs-site"]["properties"] and \
+ services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"].lower() == 'true'):
+ validationItems.append({"config-name": "dfs.allow.truncate",
+ "item": self.getWarnItem("HAWQ requires dfs.allow.truncate in hdfs-site.xml set to True.")})
+
validationProblems = self.toConfigurationValidationProblems(validationItems, "hdfs-site")
validationProblems.extend(parentValidationProblems)
return validationProblems
http://git-wip-us.apache.org/repos/asf/ambari/blob/132c5350/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 7e59a6f..362a7cd 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1521,3 +1521,63 @@ class TestHDP23StackAdvisor(TestCase):
recommendedConfigurations = {}
self.stackAdvisor.recommendRangerConfigurations(recommendedConfigurations, clusterData, services, None)
self.assertEquals(recommendedConfigurations['ranger-admin-site']['properties']['ranger.audit.solr.zookeepers'], 'NONE')
+
+ def test_validateHDFSConfigurations(self):
+ properties = {}
+ recommendedDefaults = {}
+ configurations = {
+ "core-site": {"properties": {}},
+ }
+ services = self.load_json("services-hawq-3-hosts.json")
+ services["configurations"]["hdfs-site"] = {}
+ services["configurations"]["hdfs-site"]["properties"] = {}
+ hosts = {}
+
+ expected_warning = {
+ 'config-type':'hdfs-site',
+ 'message':'HAWQ requires dfs.allow.truncate in hdfs-site.xml set to True.',
+ 'type':'configuration',
+ 'config-name':'dfs.allow.truncate',
+ 'level':'WARN'
+ }
+
+ # Test following cases:
+ # when HAWQ is being installed and dfs.allow.truncate is not set at all, warning
+ # when HAWQ is being installed and dfs.allow.truncate is set to True, no warning
+ # when HAWQ is being installed and dfs.allow.truncate is not set to True, warning
+ # when HAWQ is not installed and dfs.allow.truncate is not set at all, no warning
+ # when HAWQ is not installed and dfs.allow.truncate is set to True, no warning
+ # when HAWQ is not installed and dfs.allow.truncate is not set to True, no warning
+ # 1
+ problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEqual(len(problems), 1)
+ self.assertEqual(problems[0], expected_warning)
+
+ # 2
+ services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "True"
+ problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEqual(len(problems), 0)
+
+ # 3
+ services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "false"
+ problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEqual(len(problems), 1)
+ self.assertEqual(problems[0], expected_warning)
+
+ del services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"]
+ servicesElementWithoutHAWQ = [service for service in services["services"] if service["StackServices"]["service_name"] != "HAWQ"]
+ services["services"] = servicesElementWithoutHAWQ
+
+ # 4
+ problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEqual(len(problems), 0)
+
+ # 5
+ services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "True"
+ problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEqual(len(problems), 0)
+
+ # 6
+ services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "false"
+ problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+ self.assertEqual(len(problems), 0)
[19/19] ambari git commit: Merge branch 'trunk' into
branch-dev-patch-upgrade
Posted by nc...@apache.org.
Merge branch 'trunk' into branch-dev-patch-upgrade
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4b36397f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4b36397f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4b36397f
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 4b36397ffd14f2afed81f0c8d16ce2d2e709530f
Parents: f40df6b 09d3e8b
Author: Nate Cole <nc...@hortonworks.com>
Authored: Tue Jan 19 13:26:47 2016 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Tue Jan 19 13:26:47 2016 -0500
----------------------------------------------------------------------
.../python/resource_management/core/shell.py | 10 +-
.../libraries/functions/ranger_functions.py | 8 +-
.../libraries/functions/ranger_functions_v2.py | 8 +-
.../server/controller/KerberosHelper.java | 41 +++-
.../server/controller/KerberosHelperImpl.java | 210 +++++++++++++++++--
.../BlueprintConfigurationProcessor.java | 5 -
.../serveraction/ServerActionExecutor.java | 8 +
.../AbstractPrepareKerberosServerAction.java | 90 +++++---
.../PrepareDisableKerberosServerAction.java | 2 +-
.../PrepareEnableKerberosServerAction.java | 2 +-
.../PrepareKerberosIdentitiesServerAction.java | 3 +-
.../upgrades/SparkShufflePropertyConfig.java | 14 +-
.../ambari/server/state/ConfigHelper.java | 7 +-
.../server/state/cluster/ClusterImpl.java | 11 +-
.../server/state/stack/upgrade/Grouping.java | 125 +++++++----
.../stack/upgrade/StageWrapperBuilder.java | 2 +-
.../ambari/server/state/stack/upgrade/Task.java | 8 +
.../server/state/stack/upgrade/TaskWrapper.java | 13 ++
.../state/stack/upgrade/TaskWrapperBuilder.java | 10 +
.../svccomphost/ServiceComponentHostImpl.java | 65 ++++--
.../topology/ClusterConfigurationRequest.java | 17 +-
.../python/ambari_server/serverConfiguration.py | 15 +-
.../main/resources/Ambari-DDL-Derby-CREATE.sql | 12 ++
.../main/resources/Ambari-DDL-MySQL-CREATE.sql | 11 +
.../main/resources/Ambari-DDL-Oracle-CREATE.sql | 11 +
.../resources/Ambari-DDL-Postgres-CREATE.sql | 12 ++
.../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql | 13 ++
.../resources/Ambari-DDL-SQLAnywhere-CREATE.sql | 10 +
.../resources/Ambari-DDL-SQLServer-CREATE.sql | 11 +
.../0.96.0.2.0/configuration/hbase-env.xml | 2 +-
.../HBASE/0.96.0.2.0/package/scripts/hbase.py | 5 +-
.../0.96.0.2.0/package/scripts/params_linux.py | 2 -
.../STORM/0.9.1.2.1/package/scripts/storm.py | 11 -
.../stacks/HDP/2.0.6/services/stack_advisor.py | 2 +-
.../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml | 5 +-
.../HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml | 29 +--
.../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml | 29 +--
.../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 31 +--
.../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 8 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 8 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 10 +-
.../stacks/HDP/2.3/role_command_order.json | 4 +-
.../stacks/HDP/2.3/services/stack_advisor.py | 8 +
.../HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml | 29 +--
.../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 31 +--
.../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 8 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 29 ++-
.../services/YARN/configuration/yarn-site.xml | 14 +-
.../stacks/HDP/2.4/services/stack_advisor.py | 78 +------
.../HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml | 29 +--
.../stacks/HDP/2.4/upgrades/upgrade-2.4.xml | 8 +-
.../StackAdvisorBlueprintProcessorTest.java | 3 +
.../server/controller/KerberosHelperTest.java | 46 +++-
.../SparkShufflePropertyConfigTest.java | 43 +---
.../ambari/server/state/ConfigHelperTest.java | 52 ++++-
.../ambari/server/state/UpgradeHelperTest.java | 168 ++++++++++++---
.../ClusterConfigurationRequestTest.java | 3 +-
.../src/test/python/TestAmbariServer.py | 38 +++-
.../stacks/2.0.6/HBASE/test_hbase_client.py | 4 -
.../stacks/2.0.6/HBASE/test_hbase_master.py | 6 -
.../2.0.6/HBASE/test_hbase_regionserver.py | 8 -
.../2.0.6/HBASE/test_phoenix_queryserver.py | 4 -
.../stacks/2.3/common/test_stack_advisor.py | 60 ++++++
.../HDP/2.0.7/services/OOZIE/metainfo.xml | 176 ++++++++++++++++
.../HDP/2.1.1/upgrades/upgrade_bucket_test.xml | 20 +-
.../upgrades/upgrade_nonrolling_new_stack.xml | 6 +-
.../stacks/HDP/2.1.1/upgrades/upgrade_test.xml | 45 +++-
ambari-web/app/controllers/main/service/item.js | 25 ++-
ambari-web/app/messages.js | 3 +-
.../configs/config_recommendation_parser.js | 9 +-
...onfig_with_override_recommendation_parser.js | 2 +-
.../mixins/common/configs/enhanced_configs.js | 13 +-
ambari-web/app/templates/main/host/details.hbs | 2 +-
ambari-web/app/views/common/controls_view.js | 7 +-
.../app/views/common/quick_view_link_view.js | 5 +-
.../test/controllers/main/service/item_test.js | 12 ++
.../config_recommendation_parser_test.js | 27 ++-
.../test/views/common/quick_link_view_test.js | 51 +++++
78 files changed, 1359 insertions(+), 613 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
----------------------------------------------------------------------
[04/19] ambari git commit: AMBARI-14711. Message displayed on hover
on the host status indicator is half visible (alexantonenko)
Posted by nc...@apache.org.
AMBARI-14711. Message displayed on hover on the host status indicator is half visible (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/36af9bba
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/36af9bba
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/36af9bba
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 36af9bba684278f40c93eb7e80687f9bcec8f86f
Parents: 8684e45
Author: Alex Antonenko <hi...@gmail.com>
Authored: Mon Jan 18 19:26:34 2016 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Mon Jan 18 19:33:05 2016 +0200
----------------------------------------------------------------------
ambari-web/app/templates/main/host/details.hbs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/36af9bba/ambari-web/app/templates/main/host/details.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/main/host/details.hbs b/ambari-web/app/templates/main/host/details.hbs
index eadf8a3..f709612 100644
--- a/ambari-web/app/templates/main/host/details.hbs
+++ b/ambari-web/app/templates/main/host/details.hbs
@@ -20,7 +20,7 @@
<div id="host-details">
<div class="host-details-header">
<div class="status-info">
- <span rel="HealthTooltip" {{bindAttr class="view.content.healthClass view.content.healthIconClass"}} {{bindAttr data-original-title="view.content.healthToolTip" }}></span><span
+ <span rel="HealthTooltip" {{bindAttr class="view.content.healthClass view.content.healthIconClass"}} data-placement="bottom" {{bindAttr data-original-title="view.content.healthToolTip" }}></span><span
class='host-title'>{{unbound view.content.hostName}}</span>
{{#unless view.isActive}}
<span class="host-maintenance-notice pull-right"><span class="icon-medkit"></span> {{t hosts.host.passive.mode}}</span>