You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/01/19 19:26:56 UTC

[01/19] ambari git commit: AMBARI-14709. RU: HBase service check failed during upgrade 2.2.8->2.4 (aonishuk)

Repository: ambari
Updated Branches:
  refs/heads/branch-dev-patch-upgrade f40df6b57 -> 4b36397ff


AMBARI-14709. RU: HBase service check failed during upgrade 2.2.8->2.4 (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/40aa6121
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/40aa6121
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/40aa6121

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 40aa6121fa9cc0a3a39f02b2fb7acbe612f426ad
Parents: 5164bc5
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Jan 18 17:02:29 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Jan 18 17:02:29 2016 +0200

----------------------------------------------------------------------
 .../src/main/python/resource_management/core/shell.py     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/40aa6121/ambari-common/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/shell.py b/ambari-common/src/main/python/resource_management/core/shell.py
index 5acbda9..fac264c 100644
--- a/ambari-common/src/main/python/resource_management/core/shell.py
+++ b/ambari-common/src/main/python/resource_management/core/shell.py
@@ -141,7 +141,7 @@ def _call_wrapper(command, **kwargs):
         break
       except ExecuteTimeoutException as ex:     
         if on_timeout:
-          Logger.info("Executing '%s'. Reason: %s" % (on_timeout, str(ex)))
+          Logger.info("Executing '{0}'. Reason: {1}".format(on_timeout, str(ex)))
           result = checked_call(on_timeout)
         else:
           raise
@@ -149,7 +149,7 @@ def _call_wrapper(command, **kwargs):
       if is_last_try: # last try
         raise
       else:
-        Logger.info("Retrying after %d seconds. Reason: %s" % (try_sleep, str(ex)))
+        Logger.info("Retrying after {0} seconds. Reason: {1}".format(try_sleep, str(ex)))
         time.sleep(try_sleep)
       
   return result
@@ -283,13 +283,13 @@ def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE
       t.cancel()
     # timeout occurred
     else:
-      err_msg = Logger.filter_text(("Execution of '%s' was killed due timeout after %d seconds") % (command_alias, timeout))
+      err_msg = "Execution of '{0}' was killed due timeout after {1} seconds".format(command, timeout)
       raise ExecuteTimeoutException(err_msg)
    
   code = proc.returncode
   
   if throw_on_failure and code:
-    err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_alias, code, all_output))
+    err_msg = Logger.filter_text("Execution of '{0}' returned {1}. {2}".format(command_alias, code, all_output))
     raise Fail(err_msg)
   
   # if separate stderr is enabled (by default it's redirected to out)
@@ -313,7 +313,7 @@ def as_sudo(command, env=None, auto_escape=True):
     #   
     # In that case while passing string,
     # any bash symbols eventually added to command like && || ; < > | << >> would cause problems.
-    err_msg = Logger.filter_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple of arguments") % (command))
+    err_msg = Logger.filter_text("String command '{0}' cannot be run as sudo. Please supply the command as a tuple of arguments".format(command))
     raise Fail(err_msg)
 
   env = _get_environment_str(_add_current_path_to_env(env)) if env else ENV_PLACEHOLDER


[13/19] ambari git commit: AMBARI-14716 Prevent the user from removing the last service. (atkach)

Posted by nc...@apache.org.
AMBARI-14716 Prevent the user from removing the last service. (atkach)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/39400f4e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/39400f4e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/39400f4e

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 39400f4e97233a9d19391a28a0e47008a270d316
Parents: 869e085
Author: Andrii Tkach <at...@hortonworks.com>
Authored: Tue Jan 19 11:37:45 2016 +0200
Committer: Andrii Tkach <at...@hortonworks.com>
Committed: Tue Jan 19 11:37:45 2016 +0200

----------------------------------------------------------------------
 ambari-web/app/controllers/main/service/item.js | 25 +++++++++++++-------
 ambari-web/app/messages.js                      |  3 ++-
 .../test/controllers/main/service/item_test.js  | 12 ++++++++++
 3 files changed, 31 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/39400f4e/ambari-web/app/controllers/main/service/item.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/item.js b/ambari-web/app/controllers/main/service/item.js
index 260b1d2..2c1d4cc 100644
--- a/ambari-web/app/controllers/main/service/item.js
+++ b/ambari-web/app/controllers/main/service/item.js
@@ -954,26 +954,35 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow
    */
   deleteService: function(serviceName) {
     var dependentServices = App.StackService.find(serviceName).get('requiredServices').filter(function(_serviceName) {
-      return App.Service.find(_serviceName).get('isLoaded');
-    });
-    var self = this;
-    var displayName = App.format.role(serviceName);
-
-    if (dependentServices.length > 0) {
+          return App.Service.find(_serviceName).get('isLoaded');
+        }),
+        self = this,
+        displayName = App.format.role(serviceName),
+        popupHeader = Em.I18n.t('services.service.delete.popup.header');
+
+    if (App.Service.find().get('length') === 1) {
+      //at least one service should be installed
+      App.ModalPopup.show({
+        secondary: null,
+        header: popupHeader,
+        encodeBody: false,
+        body: Em.I18n.t('services.service.delete.lastService.popup.body').format(displayName)
+      });
+    } else if (dependentServices.length > 0) {
       this.dependentServicesWarning(serviceName, dependentServices);
     } else if (App.Service.find(serviceName).get('workStatus') === 'INSTALLED') {
       App.showConfirmationPopup(
         function() {self.confirmDeleteService(serviceName)},
         Em.I18n.t('services.service.delete.popup.warning').format(displayName),
         null,
-        Em.I18n.t('services.service.delete.popup.header'),
+        popupHeader,
         Em.I18n.t('common.delete'),
         true
       );
     } else {
       App.ModalPopup.show({
         secondary: null,
-        header: Em.I18n.t('services.service.delete.popup.header'),
+        header: popupHeader,
         encodeBody: false,
         body: Em.I18n.t('services.service.delete.popup.mustBeStopped').format(displayName)
       });

http://git-wip-us.apache.org/repos/asf/ambari/blob/39400f4e/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index 203e7ee..82797a7 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -1679,10 +1679,11 @@ Em.I18n.translations = {
   'services.service.actions.serviceActions':'Service Actions',
 
   'services.service.delete.popup.header': 'Delete Service',
+  'services.service.delete.lastService.popup.body': 'The <b>{0}</b> service can\'t be deleted, at least one service must be installed.',
   'services.service.delete.popup.dependentServices': 'Prior to deleting <b>{0}</b>, you must delete the following dependent services:',
   'services.service.delete.popup.mustBeStopped': 'Prior to deleting <b>{0}</b>, you must stop the service.',
   'services.service.delete.popup.warning': 'The <b>{0} service will be removed from Ambari and all configurations' +
-  ' and configuration history will be lost</b>',
+  ' and configuration history will be lost.</b>',
   'services.service.confirmDelete.popup.header': 'Confirm Delete',
   'services.service.confirmDelete.popup.body': 'You must confirm delete of <b>{0}</b> by typing "yes"' +
   ' in the confirmation box. <b>This operation is not reversible and all configuration history will be lost.</b>',

http://git-wip-us.apache.org/repos/asf/ambari/blob/39400f4e/ambari-web/test/controllers/main/service/item_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/main/service/item_test.js b/ambari-web/test/controllers/main/service/item_test.js
index 98ff890..55d27f7 100644
--- a/ambari-web/test/controllers/main/service/item_test.js
+++ b/ambari-web/test/controllers/main/service/item_test.js
@@ -1197,6 +1197,18 @@ describe('App.MainServiceItemController', function () {
       App.format.role.restore();
     });
 
+    it("only one service installed", function() {
+      this.mockStackService.returns(Em.Object.create({requiredServices: ['S2']}));
+      this.mockService.returns(Em.Object.create({length: 1}));
+      mainServiceItemController.deleteService('S1');
+      expect(App.ModalPopup.show.calledWith({
+        secondary: null,
+        header: Em.I18n.t('services.service.delete.popup.header'),
+        encodeBody: false,
+        body: Em.I18n.t('services.service.delete.lastService.popup.body').format('S1')
+      })).to.be.true;
+    });
+
     it("service has installed dependent services", function() {
       this.mockStackService.returns(Em.Object.create({requiredServices: ['S2']}));
       this.mockService.returns(Em.Object.create({workStatus: 'INSTALLED', isLoaded: true}));


[15/19] ambari git commit: AMBARI-12430. Improve checks for ambari.properties file for missing/needed properties during startup (aonishuk)

Posted by nc...@apache.org.
AMBARI-12430. Improve checks for ambari.properties file for missing/needed properties during startup (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/48246c9f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/48246c9f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/48246c9f

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 48246c9fbdbe13b73e9851df58c1813fc134f635
Parents: 1c2c6a9
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Jan 19 15:48:11 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Jan 19 15:48:11 2016 +0200

----------------------------------------------------------------------
 .../python/ambari_server/serverConfiguration.py | 15 ++++++--
 .../src/test/python/TestAmbariServer.py         | 38 ++++++++++++++++++--
 2 files changed, 48 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/48246c9f/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index aeb4d5b..d5b9e86 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -177,10 +177,20 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s
 
 DEFAULT_DB_NAME = "ambari"
 
+SECURITY_KEYS_DIR = "security.server.keys_dir"
 COMMON_SERVICES_PATH_PROPERTY = 'common.services.path'
 WEBAPP_DIR_PROPERTY = 'webapp.dir'
-REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY, SERVER_VERSION_FILE_PATH, \
-                       WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY]
+SHARED_RESOURCES_DIR = 'shared.resources.dir'
+BOOTSTRAP_SCRIPT = 'bootstrap.script'
+CUSTOM_ACTION_DEFINITIONS = 'custom.action.definitions'
+BOOTSTRAP_SETUP_AGENT_SCRIPT = 'bootstrap.setup_agent.script'
+STACKADVISOR_SCRIPT = 'stackadvisor.script'
+REQUIRED_PROPERTIES = [OS_FAMILY_PROPERTY, OS_TYPE_PROPERTY, COMMON_SERVICES_PATH_PROPERTY, SERVER_VERSION_FILE_PATH,
+                       WEBAPP_DIR_PROPERTY, STACK_LOCATION_KEY, SECURITY_KEYS_DIR, JDBC_DATABASE_NAME_PROPERTY,
+                       NR_USER_PROPERTY, JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY,
+                       JDBC_PASSWORD_PROPERTY, SHARED_RESOURCES_DIR, JDBC_USER_NAME_PROPERTY, BOOTSTRAP_SCRIPT,
+                       RESOURCES_DIR_PROPERTY, CUSTOM_ACTION_DEFINITIONS, BOOTSTRAP_SETUP_AGENT_SCRIPT,
+                       STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY]
 
 class ServerDatabaseType(object):
   internal = 0
@@ -457,7 +467,6 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
 configDefaults = ServerConfigDefaults()
 
 # Security
-SECURITY_KEYS_DIR = "security.server.keys_dir"
 SECURITY_MASTER_KEY_LOCATION = "security.master.key.location"
 SECURITY_KEY_IS_PERSISTED = "security.master.key.ispersisted"
 SECURITY_KEY_ENV_VAR_NAME = "AMBARI_SECURITY_MASTER_KEY"

http://git-wip-us.apache.org/repos/asf/ambari/blob/48246c9f/ambari-server/src/test/python/TestAmbariServer.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestAmbariServer.py b/ambari-server/src/test/python/TestAmbariServer.py
index 970bcab..ad4505a 100644
--- a/ambari-server/src/test/python/TestAmbariServer.py
+++ b/ambari-server/src/test/python/TestAmbariServer.py
@@ -76,7 +76,9 @@ with patch("platform.linux_distribution", return_value = os_distro_value):
           SSL_TRUSTSTORE_PASSWORD_PROPERTY, SECURITY_IS_ENCRYPTION_ENABLED, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
           SECURITY_MASTER_KEY_LOCATION, SECURITY_KEYS_DIR, LDAP_PRIMARY_URL_PROPERTY, store_password_file, \
           get_pass_file_path, GET_FQDN_SERVICE_URL, JDBC_USE_INTEGRATED_AUTH_PROPERTY, SECURITY_KEY_ENV_VAR_NAME, \
-          JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY
+          JAVA_HOME_PROPERTY, JDK_NAME_PROPERTY, JCE_NAME_PROPERTY, STACK_LOCATION_KEY, SERVER_VERSION_FILE_PATH, \
+          COMMON_SERVICES_PATH_PROPERTY, WEBAPP_DIR_PROPERTY, SHARED_RESOURCES_DIR, BOOTSTRAP_SCRIPT, \
+          CUSTOM_ACTION_DEFINITIONS, BOOTSTRAP_SETUP_AGENT_SCRIPT, STACKADVISOR_SCRIPT, BOOTSTRAP_DIR_PROPERTY
         from ambari_server.serverUtils import is_server_runing, refresh_stack_hash
         from ambari_server.serverSetup import check_selinux, check_ambari_user, proceedJDBCProperties, SE_STATUS_DISABLED, SE_MODE_ENFORCING, configure_os_settings, \
           download_and_install_jdk, prompt_db_properties, setup, \
@@ -4348,6 +4350,7 @@ class TestAmbariServer(TestCase):
       del args.jdbc_url
       del args.debug
       del args.suspend_start
+      args.skip_properties_validation = False
 
       return args
 
@@ -4368,6 +4371,27 @@ class TestAmbariServer(TestCase):
 
     p = Properties()
     p.process_pair(SECURITY_IS_ENCRYPTION_ENABLED, 'False')
+    p.process_pair(JDBC_DATABASE_NAME_PROPERTY, 'some_value')
+    p.process_pair(NR_USER_PROPERTY, 'some_value')
+    p.process_pair(STACK_LOCATION_KEY, 'some_value')
+    p.process_pair(SERVER_VERSION_FILE_PATH, 'some_value')
+    p.process_pair(OS_TYPE_PROPERTY, 'some_value')
+    p.process_pair(JAVA_HOME_PROPERTY, 'some_value')
+    p.process_pair(JDK_NAME_PROPERTY, 'some_value')
+    p.process_pair(JCE_NAME_PROPERTY, 'some_value')
+    p.process_pair(COMMON_SERVICES_PATH_PROPERTY, 'some_value')
+    p.process_pair(JDBC_PASSWORD_PROPERTY, 'some_value')
+    p.process_pair(WEBAPP_DIR_PROPERTY, 'some_value')
+    p.process_pair(SHARED_RESOURCES_DIR, 'some_value')
+    p.process_pair(SECURITY_KEYS_DIR, 'some_value')
+    p.process_pair(JDBC_USER_NAME_PROPERTY, 'some_value')
+    p.process_pair(BOOTSTRAP_SCRIPT, 'some_value')
+    p.process_pair(OS_FAMILY_PROPERTY, 'some_value')
+    p.process_pair(RESOURCES_DIR_PROPERTY, 'some_value')
+    p.process_pair(CUSTOM_ACTION_DEFINITIONS, 'some_value')
+    p.process_pair(BOOTSTRAP_SETUP_AGENT_SCRIPT, 'some_value')
+    p.process_pair(STACKADVISOR_SCRIPT, 'some_value')
+    p.process_pair(BOOTSTRAP_DIR_PROPERTY, 'some_value')
 
     get_ambari_properties_5_mock.return_value = get_ambari_properties_4_mock.return_value = \
       get_ambari_properties_3_mock.return_value = get_ambari_properties_2_mock.return_value = \
@@ -4682,7 +4706,17 @@ class TestAmbariServer(TestCase):
     self.assertTrue(save_master_key_method.called)
     popen_arg = popenMock.call_args[1]['env']
     self.assertEquals(os_environ_mock.copy.return_value, popen_arg)
-    pass
+
+    # Checking situation when required properties not set up
+    args = reset_mocks()
+    p.removeProp(JAVA_HOME_PROPERTY)
+    get_ambari_properties_mock.return_value = p
+    try:
+      _ambari_server_.start(args)
+      self.fail("Should fail with 'Required properties are not found:'")
+    except FatalException as e:
+      # Expected
+      self.assertTrue('Required properties are not found:' in e.reason)
 
 
   @not_for_platform(PLATFORM_WINDOWS)


[06/19] ambari git commit: AMBARI-14695: HAWQSTANDBY fails to start when HAWQ Service is started (mithmatt via jaoki)

Posted by nc...@apache.org.
AMBARI-14695: HAWQSTANDBY fails to start when HAWQ Service is started (mithmatt via jaoki)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/342363ff
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/342363ff
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/342363ff

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 342363ff0d2192eea3e87949adad1c725a4b871e
Parents: 0cbefb3
Author: Jun Aoki <ja...@apache.org>
Authored: Mon Jan 18 11:06:24 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Mon Jan 18 11:06:24 2016 -0800

----------------------------------------------------------------------
 .../src/main/resources/stacks/HDP/2.3/role_command_order.json    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/342363ff/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
index 2c44387..1b6eefc 100755
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
@@ -10,8 +10,8 @@
     "PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"],
     "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"],
     "SPARK_THRIFTSERVER-START" : ["NAMENODE-START", "HIVE_METASTORE-START"],
-    "HAWQSTANDBY-START" : ["NAMENODE-START", "DATANODE-START"],
-    "HAWQMASTER-START" : ["NAMENODE-START", "DATANODE-START", "HAWQSTANDBY-START"],
+    "HAWQMASTER-START" : ["NAMENODE-START", "DATANODE-START"],
+    "HAWQSTANDBY-START" : ["NAMENODE-START", "DATANODE-START", "HAWQMASTER-START"],
     "HAWQSEGMENT-START" : ["HAWQMASTER-START", "HAWQSTANDBY-START"],
     "HAWQ_SERVICE_CHECK-SERVICE_CHECK" : ["HAWQMASTER-START", "HDFS_SERVICE_CHECK-SERVICE_CHECK"],
     "KNOX_GATEWAY-START" : ["RANGER_USERSYNC-START", "NAMENODE-START"],


[14/19] ambari git commit: AMBARI-14718. on region server restart ambari is setting /tmp to hbase:hadoop (aonishuk)

Posted by nc...@apache.org.
AMBARI-14718. on region server restart ambari is setting /tmp to hbase:hadoop (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1c2c6a9f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1c2c6a9f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1c2c6a9f

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 1c2c6a9f6678931936585761a242651b8a6871ce
Parents: 39400f4
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Jan 19 14:15:03 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Jan 19 14:15:03 2016 +0200

----------------------------------------------------------------------
 .../HBASE/0.96.0.2.0/configuration/hbase-env.xml             | 2 +-
 .../HBASE/0.96.0.2.0/package/scripts/hbase.py                | 5 ++---
 .../HBASE/0.96.0.2.0/package/scripts/params_linux.py         | 2 --
 .../src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py  | 4 ----
 .../src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py  | 6 ------
 .../python/stacks/2.0.6/HBASE/test_hbase_regionserver.py     | 8 --------
 .../python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py    | 4 ----
 7 files changed, 3 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
index cabd4fd..62c7c93 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/configuration/hbase-env.xml
@@ -115,7 +115,7 @@ and the -Xmn ratio (hbase_regionserver_xmn_ratio) exceeds this value.
 
   <property>
     <name>hbase_java_io_tmpdir</name>
-    <value>/tmp</value>
+    <value>{{exec_tmp_dir}}/hbase_java_io_tmpdir</value>
     <description>Used in hbase-env.sh as HBASE_OPTS=-Djava.io.tmpdir=java_io_tmpdir</description>
     <value-attributes>
       <type>directory</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
index c407a93..71ce02b 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase.py
@@ -22,6 +22,7 @@ from resource_management import *
 import sys
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons import OSConst
+from resource_management.core.source import InlineTemplate
 
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
@@ -56,9 +57,7 @@ def hbase(name=None):
       create_parents = True
   )
    
-  Directory(params.java_io_tmpdir,
-      owner = params.hbase_user,
-      group = params.user_group,
+  Directory(InlineTemplate(params.java_io_tmpdir).get_content(),
       create_parents = True,
       mode=0777
   )

http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index 7679d90..4a44676 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -36,8 +36,6 @@ from resource_management.libraries.functions import is_empty
 from resource_management.libraries.functions import get_unique_id_and_date
 from resource_management.libraries.script.script import Script
 
-from resource_management.libraries.functions.substitute_vars import substitute_vars
-
 # server configurations
 config = Script.get_config()
 exec_tmp_dir = Script.get_tmp_dir()

http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
index 9f3e19e..6dc4b0b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py
@@ -47,8 +47,6 @@ class TestHBaseClient(RMFTestCase):
       create_parents = True,
     )
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )
@@ -145,8 +143,6 @@ class TestHBaseClient(RMFTestCase):
       create_parents = True,
     )
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )

http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
index 7c6d10e..13b2e33 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
@@ -243,8 +243,6 @@ class TestHBaseMaster(RMFTestCase):
       create_parents = True,
     )
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )
@@ -380,8 +378,6 @@ class TestHBaseMaster(RMFTestCase):
       create_parents = True,
     )
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )
@@ -526,8 +522,6 @@ class TestHBaseMaster(RMFTestCase):
       group = 'hadoop',
       create_parents = True)
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )

http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
index c36bd89..58ca7ef 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py
@@ -136,8 +136,6 @@ class TestHbaseRegionServer(RMFTestCase):
       create_parents = True,
     )
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )
@@ -234,8 +232,6 @@ class TestHbaseRegionServer(RMFTestCase):
       create_parents = True,
     )
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )
@@ -342,8 +338,6 @@ class TestHbaseRegionServer(RMFTestCase):
       group = 'hadoop',
       create_parents = True)
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )
@@ -458,8 +452,6 @@ class TestHbaseRegionServer(RMFTestCase):
       group = 'hadoop',
       create_parents = True)
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )

http://git-wip-us.apache.org/repos/asf/ambari/blob/1c2c6a9f/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
index 6f3199e..7722af0 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py
@@ -238,8 +238,6 @@ class TestPhoenixQueryServer(RMFTestCase):
       create_parents = True,
     )
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )
@@ -351,8 +349,6 @@ class TestPhoenixQueryServer(RMFTestCase):
       create_parents = True,
     )
     self.assertResourceCalled('Directory', '/tmp',
-      owner = 'hbase',
-      group = 'hadoop',
       create_parents = True,
       mode = 0777
     )


[07/19] ambari git commit: AMBARI-14653. HBase Links Metric - more links pointing to the wrong hbase.master.info.port. (Zhe (Joe) Wang via Jaimin)

Posted by nc...@apache.org.
AMBARI-14653. HBase Links Metric - more links pointing to the wrong hbase.master.info.port. (Zhe (Joe) Wang via Jaimin)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/21b6b316
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/21b6b316
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/21b6b316

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 21b6b31696718f5190a5b2cd52c469b097e00ffa
Parents: 342363f
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Mon Jan 18 11:20:58 2016 -0800
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Mon Jan 18 11:20:58 2016 -0800

----------------------------------------------------------------------
 .../app/views/common/quick_view_link_view.js    |  5 +-
 .../test/views/common/quick_link_view_test.js   | 51 ++++++++++++++++++++
 2 files changed, 53 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/21b6b316/ambari-web/app/views/common/quick_view_link_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/quick_view_link_view.js b/ambari-web/app/views/common/quick_view_link_view.js
index e58e47f..604d24d 100644
--- a/ambari-web/app/views/common/quick_view_link_view.js
+++ b/ambari-web/app/views/common/quick_view_link_view.js
@@ -163,7 +163,7 @@ App.QuickViewLinks = Em.View.extend({
       var sites = ['core-site', 'hdfs-site'];
       if(checks){
         checks.forEach(function(check){
-          var protocolConfigSiteProp = Em.get(check, 'site')
+          var protocolConfigSiteProp = Em.get(check, 'site');
           if (sites.indexOf(protocolConfigSiteProp) < 0){
             sites.push(protocolConfigSiteProp);
           }
@@ -181,8 +181,7 @@ App.QuickViewLinks = Em.View.extend({
             }
           }
         }, this);
-
-        this.set('requiredSiteNames', sites);
+        this.set('requiredSiteNames', this.get('requiredSiteNames').pushObjects(sites).uniq());
         this.setQuickLinks();
       }
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/21b6b316/ambari-web/test/views/common/quick_link_view_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/common/quick_link_view_test.js b/ambari-web/test/views/common/quick_link_view_test.js
index cae4715..b9aecee 100644
--- a/ambari-web/test/views/common/quick_link_view_test.js
+++ b/ambari-web/test/views/common/quick_link_view_test.js
@@ -180,6 +180,57 @@ describe('App.QuickViewLinks', function () {
     });
   });
 
+  describe("#loadQuickLinksConfigSuccessCallback()", function () {
+    var mock;
+
+    beforeEach(function () {
+      sinon.stub(App.store, 'commit', Em.K);
+      mock = sinon.stub(quickViewLinks, 'getQuickLinksConfiguration');
+    });
+    afterEach(function () {
+      App.store.commit.restore();
+      mock.restore();
+    });
+    it("requiredSites consistent", function () {
+      var quickLinksConfigHBASE = {
+        protocol: {
+          type: "http"
+        },
+        links: [
+          {
+            port: {
+              site: "hbase-site"
+            }
+          }
+        ]
+      };
+      var quickLinksConfigYARN = {
+        protocol: {
+          checks: [
+            {
+              site: "yarn-site"
+            }
+          ],
+          type: "https"
+        },
+        links: [
+          {
+            port: {
+              site: "yarn-site"
+            }
+          }
+        ]
+      };
+      quickViewLinks.set('content.serviceName', 'HBASE');
+      mock.returns(quickLinksConfigHBASE);
+      quickViewLinks.loadQuickLinksConfigSuccessCallback({items: []});
+      quickViewLinks.set('content.serviceName', 'YARN');
+      mock.returns(quickLinksConfigYARN);
+      quickViewLinks.loadQuickLinksConfigSuccessCallback({items: []});
+      expect(quickViewLinks.get('requiredSiteNames')).to.be.eql(["core-site", "hdfs-site", "hbase-site", "yarn-site"]);
+    });
+  });
+
   describe("#getQuickLinksHosts()", function () {
     beforeEach(function () {
       sinon.stub(App.ajax, 'send');


[16/19] ambari git commit: AMBARI-14721. yarn-site properties for Spark Shuffle Aux should be added unconditionally (aonishuk)

Posted by nc...@apache.org.
AMBARI-14721. yarn-site properties for Spark Shuffle Aux should be added unconditionally (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8f6b45d5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8f6b45d5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8f6b45d5

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8f6b45d5a13179cc49d0d4a857c5c157d757bb11
Parents: 48246c9
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Jan 19 17:03:11 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Jan 19 17:03:11 2016 +0200

----------------------------------------------------------------------
 .../upgrades/SparkShufflePropertyConfig.java    | 14 ++--
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml |  2 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml     |  2 +-
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml |  2 +-
 .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml     |  2 +-
 .../services/YARN/configuration/yarn-site.xml   | 14 +---
 .../stacks/HDP/2.4/services/stack_advisor.py    | 78 +-------------------
 .../SparkShufflePropertyConfigTest.java         | 43 +----------
 8 files changed, 12 insertions(+), 145 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
index efeb4a0..299a373 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
@@ -68,9 +68,6 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
       return  createCommandReport(0, HostRoleStatus.FAILED,"{}",
               String.format("Source type %s not found", YARN_SITE_CONFIG_TYPE), "");
     }
-
-    try {
-      cluster.getService("SPARK");  // check if SPARK service present
       Map<String, String> yarnSiteProperties = yarnSiteConfig.getProperties();
 
       final List<String> auxSevices;
@@ -82,7 +79,11 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
       } else {
         auxSevices = new ArrayList<>();
       }
-      auxSevices.add(SPARK_SHUFFLE_AUX_STR);
+
+      // check if spark is not already in the list
+      if (!auxSevices.contains(SPARK_SHUFFLE_AUX_STR)) {
+        auxSevices.add(SPARK_SHUFFLE_AUX_STR);
+      }
       newAuxServices = StringUtils.join(auxSevices, ",");
 
       yarnSiteProperties.put(YARN_NODEMANAGER_AUX_SERVICES, newAuxServices);
@@ -94,10 +95,5 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
         String.format("%s was set from %s to %s. %s was set to %s",
                 YARN_NODEMANAGER_AUX_SERVICES, oldAuxServices, newAuxServices,
                 YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE, YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE), "");
-
-    } catch (ServiceNotFoundException e) {
-      return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
-          String.format("%s not updated as no SPARK service present on the cluster", YARN_NODEMANAGER_AUX_SERVICES), "");
-    }
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index ff7e032..c5d74e2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -395,7 +395,7 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark Shuffle">
         <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
       </execute-stage>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index 34e4848..6b5bcb1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -566,7 +566,7 @@
 
           <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
 
-          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index f58eb79..a3880b3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -269,7 +269,7 @@
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
 
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
-        <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+        <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
       </execute-stage>
 
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 573713c..c15769b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -488,7 +488,7 @@
 
       <component name="RESOURCEMANAGER">
         <pre-upgrade>
-          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
         </pre-upgrade>
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
index 134d9b0..5e8eae6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
@@ -20,26 +20,14 @@
 <configuration supports_final="true">
     <property>
         <name>yarn.nodemanager.aux-services</name>
-        <value>mapreduce_shuffle</value>
+        <value>mapreduce_shuffle,spark_shuffle</value>
         <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
           not start with numbers</description>
-        <depends-on>
-            <property>
-              <type>spark-default</type>
-              <name>spark.yarn.queue</name>
-            </property>
-        </depends-on>
     </property>
 
     <property>
         <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
         <value>org.apache.spark.network.yarn.YarnShuffleService</value>
         <description>The auxiliary service class to use for Spark</description>
-        <depends-on>
-            <property>
-                <type>spark-default</type>
-                <name>spark.yarn.queue</name>
-            </property>
-        </depends-on>
     </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
index 3031ab3..985c101 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
@@ -19,80 +19,4 @@ limitations under the License.
 
 
 class HDP24StackAdvisor(HDP23StackAdvisor):
-  def getServiceConfigurationRecommenderDict(self):
-    parent_recommend_conf_dict = super(HDP24StackAdvisor, self).getServiceConfigurationRecommenderDict()
-    child_recommend_conf_dict = {
-      "YARN": self.recommendYARNConfigurations
-    }
-    parent_recommend_conf_dict.update(child_recommend_conf_dict)
-    return parent_recommend_conf_dict
-
-  def getServiceConfigurationValidators(self):
-    parentValidators = super(HDP24StackAdvisor, self).getServiceConfigurationValidators()
-    childValidators = {
-      "YARN": {"yarn-site": self.validateYARNConfigurations}
-    }
-    self.mergeValidators(parentValidators, childValidators)
-    return parentValidators
-
-  def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
-    super(HDP24StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
-
-    yarn_site_config = "yarn-site"
-    properties = services["configurations"] if yarn_site_config in services["configurations"] else []
-    yarn_site_properties = properties[yarn_site_config]["properties"] if yarn_site_config in properties and \
-                                                                         "properties" in properties[yarn_site_config] else []
-    put_yarn_site_property = self.putProperty(configurations, yarn_site_config, services)
-    put_yarn_site_property_attributes = self.putPropertyAttribute(configurations, yarn_site_config)
-    services_list = [service["StackServices"]["service_name"] for service in services["services"]]
-
-    if 'SPARK' in services_list:
-      if "yarn.nodemanager.aux-services" in yarn_site_properties:
-        aux_services = yarn_site_properties["yarn.nodemanager.aux-services"].split(",")
-        if "spark_shuffle" not in aux_services:
-          aux_services.append("spark_shuffle")
-        put_yarn_site_property("yarn.nodemanager.aux-services", ",".join(aux_services))
-      else:
-        put_yarn_site_property("yarn.nodemanager.aux-services", "spark_shuffle")
-
-      put_yarn_site_property("yarn.nodemanager.aux-services.spark_shuffle.class",
-                             "org.apache.spark.network.yarn.YarnShuffleService")
-    else:
-      put_yarn_site_property_attributes("yarn.nodemanager.aux-services.spark_shuffle.class", "delete", "true")
-
-  def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
-    yarn_site = properties
-    validationItems = []
-    services_list = [service["StackServices"]["service_name"] for service in services["services"]]
-
-    if "SPARK" in services_list and 'YARN' in services_list:
-      # yarn.nodemanager.aux-services = ...,spark_shuffle,....
-      # yarn.nodemanager.aux-services.spark_shuffle.class = <not set>
-      if "yarn.nodemanager.aux-services" in yarn_site \
-        and "spark_shuffle" in yarn_site["yarn.nodemanager.aux-services"].lower() \
-        and "yarn.nodemanager.aux-services.spark_shuffle.class" not in yarn_site:
-        validationItems.append({
-          "config-name": "yarn.nodemanager.aux-services.spark_shuffle.class",
-          "item": self.getErrorItem("If spark_shuffle is listed in the aux-services, property value for " +
-                                    "yarn.nodemanager.aux-services.spark_shuffle.class need to be set")
-        })
-
-      # yarn.nodemanager.aux-services = <not set>
-      # yarn.nodemanager.aux-services.spark_shuffle.class = is set
-      spark_aux_service_warning = False
-      if "yarn.nodemanager.aux-services" in yarn_site and "spark_shuffle" not in yarn_site[
-        "yarn.nodemanager.aux-services"].lower() \
-        and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site:
-        spark_aux_service_warning = True
-
-      if "yarn.nodemanager.aux-services" not in yarn_site and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site:
-        spark_aux_service_warning = True
-
-      if spark_aux_service_warning:
-        validationItems.append({
-          "config-name": "yarn.nodemanager.aux-services",
-          "item": self.getWarnItem("If yarn.nodemanager.aux-services.spark_shuffle.class is set, probably " +
-                                   "aux-services property need to be updated to enable spark_shuffle")
-        })
-
-    return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+ pass

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f6b45d5/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
index edba6f0..e65a824 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
@@ -22,7 +22,6 @@ import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
@@ -30,7 +29,6 @@ import java.lang.reflect.Field;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.ambari.server.ServiceNotFoundException;
 import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
 import org.apache.ambari.server.actionmanager.HostRoleCommand;
 import org.apache.ambari.server.agent.CommandReport;
@@ -39,7 +37,6 @@ import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.ConfigImpl;
-import org.apache.ambari.server.state.Service;
 import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
@@ -94,20 +91,16 @@ public class SparkShufflePropertyConfigTest {
   }
 
   @Test
-  public void testSparkPresent() throws Exception {
+  public void testAction() throws Exception {
     ExecutionCommand executionCommand = new ExecutionCommand();
     Map<String, String> commandParams = new HashMap<>();
     HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
-    Service sparkService = EasyMock.createMock(Service.class);
     SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
 
     commandParams.put("clusterName", "c1");
     executionCommand.setCommandParams(commandParams);
     executionCommand.setClusterName("c1");
 
-    // test calculations when spark service present
-    expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce();
-
     expect(hrc.getRequestId()).andReturn(1L).anyTimes();
     expect(hrc.getStageId()).andReturn(2L).anyTimes();
     expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
@@ -131,39 +124,5 @@ public class SparkShufflePropertyConfigTest {
     assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class"));
   }
 
-  @Test
-  public void testSparkNotPresent() throws Exception{
-    SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
-    Map<String, String> commandParams = new HashMap<>();
-    ExecutionCommand executionCommand = new ExecutionCommand();
-    HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
-    ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK");
 
-    commandParams.put("clusterName", "c1");
-    executionCommand.setCommandParams(commandParams);
-    executionCommand.setClusterName("c1");
-
-    //no spark service present
-    expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce();
-
-    expect(hrc.getRequestId()).andReturn(1L).anyTimes();
-    expect(hrc.getStageId()).andReturn(2L).anyTimes();
-    expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
-    replay(cluster, hrc);
-
-    clusterField.set(action, m_clusters);
-    action.setExecutionCommand(executionCommand);
-    action.setHostRoleCommand(hrc);
-
-    CommandReport report = action.execute(null);
-    assertNotNull(report);
-
-    Cluster c = m_clusters.getCluster("c1");
-    Config config = c.getDesiredConfigByType("yarn-site");
-    Map<String, String> map = config.getProperties();
-
-    assertTrue(map.containsKey("yarn.nodemanager.aux-services"));
-    assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class"));
-    assertEquals("some_service", map.get("yarn.nodemanager.aux-services"));
-  }
 }


[10/19] ambari git commit: AMBARI-14678. RU - Oozie HA, Service Check fails since the pre-upgrade directive has multiple tasks that should run only on the specified hosts (alejandro)

Posted by nc...@apache.org.
AMBARI-14678. RU - Oozie HA, Service Check fails since the pre-upgrade directive has multiple tasks that should run only on the specified hosts (alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cbed6402
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cbed6402
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cbed6402

Branch: refs/heads/branch-dev-patch-upgrade
Commit: cbed6402604d7e80a017c358a131191706255e16
Parents: f3ffe0c
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Mon Jan 18 11:41:37 2016 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Mon Jan 18 19:17:15 2016 -0800

----------------------------------------------------------------------
 .../server/state/stack/upgrade/Grouping.java    | 125 ++++++++-----
 .../stack/upgrade/StageWrapperBuilder.java      |   2 +-
 .../ambari/server/state/stack/upgrade/Task.java |   8 +
 .../server/state/stack/upgrade/TaskWrapper.java |  13 ++
 .../state/stack/upgrade/TaskWrapperBuilder.java |  10 ++
 .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml |   5 +-
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml |  29 +--
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml |  29 +--
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml |  29 +--
 .../stacks/HDP/2.2/upgrades/upgrade-2.2.xml     |   8 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml     |   8 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml     |   8 +-
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml |  29 +--
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml |  29 +--
 .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml     |   8 +-
 .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml     |   8 +-
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml |  29 +--
 .../stacks/HDP/2.4/upgrades/upgrade-2.4.xml     |   8 +-
 .../ambari/server/state/UpgradeHelperTest.java  | 168 ++++++++++++++----
 .../HDP/2.0.7/services/OOZIE/metainfo.xml       | 176 +++++++++++++++++++
 .../HDP/2.1.1/upgrades/upgrade_bucket_test.xml  |  20 +--
 .../upgrades/upgrade_nonrolling_new_stack.xml   |   6 +-
 .../stacks/HDP/2.1.1/upgrades/upgrade_test.xml  |  45 ++++-
 23 files changed, 520 insertions(+), 280 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
index aa852bd..5af893e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Grouping.java
@@ -100,61 +100,32 @@ public class Grouping {
       boolean forUpgrade = ctx.getDirection().isUpgrade();
 
       // Construct the pre tasks during Upgrade/Downgrade direction.
+      // Buckets are grouped by the type, e.g., bucket of all Execute tasks, or all Configure tasks.
       List<TaskBucket> buckets = buckets(resolveTasks(forUpgrade, true, pc));
       for (TaskBucket bucket : buckets) {
+        // The TaskWrappers take into account if a task is meant to run on all, any, or master.
+        // A TaskWrapper may contain multiple tasks, but typically only one, and they all run on the same set of hosts.
         List<TaskWrapper> preTasks = TaskWrapperBuilder.getTaskList(service, pc.name, hostsType, bucket.tasks, params);
-        Set<String> preTasksEffectiveHosts = TaskWrapperBuilder.getEffectiveHosts(preTasks);
-        if (!preTasksEffectiveHosts.isEmpty()) {
-          StageWrapper stage = new StageWrapper(
-              bucket.type,
-              getStageText("Preparing", ctx.getComponentDisplay(service, pc.name), preTasksEffectiveHosts),
-              params,
-              preTasks
-          );
-          m_stages.add(stage);
+        List<List<TaskWrapper>> organizedTasks = organizeTaskWrappersBySyncRules(preTasks);
+        for (List<TaskWrapper> tasks : organizedTasks) {
+          addTasksToStageInBatches(tasks, "Preparing", ctx, service, pc, params);
         }
       }
 
       // Add the processing component
       if (null != pc.tasks && 1 == pc.tasks.size()) {
         Task t = pc.tasks.get(0);
-        if(m_grouping.parallelScheduler != null) {
-          List<Set<String>> hostSets = SetUtils.split(hostsType.hosts, m_grouping.parallelScheduler.maxDegreeOfParallelism);
-          int batchNum = 1;
-          for(Set<String> hosts : hostSets) {
-            // Create single stage for all
-            StageWrapper stage = new StageWrapper(
-                t.getStageWrapperType(),
-                getStageText(t.getActionVerb(), ctx.getComponentDisplay(service, pc.name), hosts, batchNum++, hostSets.size()),
-                params,
-                new TaskWrapper(service, pc.name, hosts, params, t));
-            m_stages.add(stage);
-          }
-        } else {
-          for (String hostName : hostsType.hosts) {
-            StageWrapper stage = new StageWrapper(
-                t.getStageWrapperType(),
-                getStageText(t.getActionVerb(), ctx.getComponentDisplay(service, pc.name), Collections.singleton(hostName)),
-                params,
-                new TaskWrapper(service, pc.name, Collections.singleton(hostName), params, t));
-            m_stages.add(stage);
-          }
-        }
+        TaskWrapper tw = new TaskWrapper(service, pc.name, hostsType.hosts, params, Collections.singletonList(t));
+        addTasksToStageInBatches(Collections.singletonList(tw), t.getActionVerb(), ctx, service, pc, params);
       }
 
       // Construct the post tasks during Upgrade/Downgrade direction.
       buckets = buckets(resolveTasks(forUpgrade, false, pc));
       for (TaskBucket bucket : buckets) {
         List<TaskWrapper> postTasks = TaskWrapperBuilder.getTaskList(service, pc.name, hostsType, bucket.tasks, params);
-        Set<String> postTasksEffectiveHosts = TaskWrapperBuilder.getEffectiveHosts(postTasks);
-        if (!postTasksEffectiveHosts.isEmpty()) {
-          StageWrapper stage = new StageWrapper(
-              bucket.type,
-              getStageText("Completing", ctx.getComponentDisplay(service, pc.name), postTasksEffectiveHosts),
-              params,
-              postTasks
-              );
-          m_stages.add(stage);
+        List<List<TaskWrapper>> organizedTasks = organizeTaskWrappersBySyncRules(postTasks);
+        for (List<TaskWrapper> tasks : organizedTasks) {
+          addTasksToStageInBatches(tasks, "Completing", ctx, service, pc, params);
         }
       }
 
@@ -165,6 +136,80 @@ public class Grouping {
     }
 
     /**
+     * Split a list of TaskWrappers into a list of lists where a TaskWrapper that has any task with isSequential == true
+     * must be a singleton in its own list.
+     * @param tasks List of TaskWrappers to analyze
+     * @return List of list of TaskWrappers, where each outer list is a separate stage.
+     */
+    private List<List<TaskWrapper>> organizeTaskWrappersBySyncRules(List<TaskWrapper> tasks) {
+      List<List<TaskWrapper>> groupedTasks = new ArrayList<List<TaskWrapper>>();
+
+      List<TaskWrapper> subTasks = new ArrayList<>();
+      for (TaskWrapper tw : tasks) {
+        // If an of this TaskWrapper's tasks must be on its own stage, write out the previous subtasks if possible into one complete stage.
+        if (tw.isAnyTaskSequential()) {
+          if (!subTasks.isEmpty()) {
+            groupedTasks.add(subTasks);
+            subTasks = new ArrayList<>();
+          }
+          groupedTasks.add(Collections.singletonList(tw));
+        } else {
+          subTasks.add(tw);
+        }
+      }
+      
+      if (!subTasks.isEmpty()) {
+        groupedTasks.add(subTasks);
+      }
+
+      return groupedTasks;
+    }
+
+    /**
+     * Helper function to analyze a ProcessingComponent and add its task to stages, depending on the batch size.
+     * @param tasks Collection of tasks for this stage
+     * @param verb Verb string to use in the title of the task
+     * @param ctx Upgrade Context
+     * @param service Service
+     * @param pc Processing Component
+     * @param params Params to add to the stage.
+     */
+    private void addTasksToStageInBatches(List<TaskWrapper> tasks, String verb, UpgradeContext ctx, String service, ProcessingComponent pc, Map<String, String> params) {
+      if (tasks == null || tasks.isEmpty() || tasks.get(0).getTasks() == null || tasks.get(0).getTasks().isEmpty()) {
+        return;
+      }
+
+      // Our assumption is that all of the tasks in the StageWrapper are of the same type.
+      StageWrapper.Type type = tasks.get(0).getTasks().get(0).getStageWrapperType();
+
+      // Expand some of the TaskWrappers into multiple based on the batch size.
+      for (TaskWrapper tw : tasks) {
+        List<Set<String>> hostSets = null;
+        if (m_grouping.parallelScheduler != null && m_grouping.parallelScheduler.maxDegreeOfParallelism > 0) {
+          hostSets = SetUtils.split(tw.getHosts(), m_grouping.parallelScheduler.maxDegreeOfParallelism);
+        } else {
+          hostSets = SetUtils.split(tw.getHosts(), 1);
+        }
+
+        int numBatchesNeeded = hostSets.size();
+        int batchNum = 0;
+        for (Set<String> hostSubset : hostSets) {
+          batchNum++;
+          TaskWrapper expandedTW = new TaskWrapper(tw.getService(), tw.getComponent(), hostSubset, tw.getParams(), tw.getTasks());
+
+          String stageText = getStageText(verb, ctx.getComponentDisplay(service, pc.name), hostSubset, batchNum, numBatchesNeeded);
+
+          StageWrapper stage = new StageWrapper(
+              type,
+              stageText,
+              params,
+              new TaskWrapper(service, pc.name, hostSubset, params, tw.getTasks()));
+          m_stages.add(stage);
+        }
+      }
+    }
+    
+    /**
      * Determine if service checks need to be ran after the stages.
      * @param upgradeContext the upgrade context
      * @return Return the stages, which may potentially be followed by service checks.

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java
index b0df83f..e96c7ff 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/StageWrapperBuilder.java
@@ -169,7 +169,7 @@ public abstract class StageWrapperBuilder {
    */
   protected String getStageText(String prefix, String component, Set<String> hosts, int batchNum, int totalBatches) {
     String stageText = getStageText(prefix, component, hosts);
-    String batchText = 1 == totalBatches? "" : String.format(" ( Batch %s of %s )", batchNum, totalBatches);
+    String batchText = 1 == totalBatches? "" : String.format(" (Batch %s of %s)", batchNum, totalBatches);
     return stageText + batchText;
   }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
index 980f88e..f365a7f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/Task.java
@@ -17,6 +17,7 @@
  */
 package org.apache.ambari.server.state.stack.upgrade;
 
+import javax.xml.bind.annotation.XmlAttribute;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.XmlSeeAlso;
 
@@ -34,6 +35,13 @@ public abstract class Task {
   public String summary;
 
   /**
+   * Whether the task needs to run sequentially, i.e., on its own stage.
+   * If false, will be grouped with other tasks.
+   */
+  @XmlAttribute(name = "sequential")
+  public boolean isSequential = false;
+
+  /**
    * @return the type of the task
    */
   public abstract Type getType();

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java
index 69b3f8b..5fdf91c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapper.java
@@ -112,4 +112,17 @@ public class TaskWrapper {
     return component;
   }
 
+  /**
+   * @return true if any task is sequential, otherwise, return false.
+   */
+  public boolean isAnyTaskSequential() {
+    for (Task t : getTasks()) {
+      if (t.isSequential) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
index 057c310..81a3a4d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.ambari.server.stack.HostsType;
+import org.apache.ambari.server.utils.StageUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -47,8 +48,16 @@ public class TaskWrapperBuilder {
    * @param params additional parameters
    */
   public static List<TaskWrapper> getTaskList(String service, String component, HostsType hostsType, List<Task> tasks, Map<String, String> params) {
+    // Ok if Ambari Server is not part of the cluster hosts since this is only used in the calculation of how many batches
+    // to create.
+    String ambariServerHostname = StageUtils.getHostName();
+
     List<TaskWrapper> collection = new ArrayList<TaskWrapper>();
     for (Task t : tasks) {
+      if (t.getType().equals(Task.Type.CONFIGURE) || t.getType().equals(Task.Type.MANUAL)) {
+        collection.add(new TaskWrapper(service, component, Collections.singleton(ambariServerHostname), params, t));
+        continue;
+      }
       if (t.getType().equals(Task.Type.EXECUTE)) {
         ExecuteTask et = (ExecuteTask) t;
         if (et.hosts == ExecuteHostType.MASTER) {
@@ -70,6 +79,7 @@ public class TaskWrapperBuilder {
             continue;
           }
         }
+        // Otherwise, meant to run on ALL hosts.
       }
 
       collection.add(new TaskWrapper(service, component, hostsType.hosts, params, t));

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
index 2cf9da9..9f0458b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml
@@ -793,14 +793,15 @@
           from HDP 2.1->2.3.
           -->
           <task xsi:type="configure_function"/>
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
index cbd9c2a..3aaf8c7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml
@@ -415,31 +415,6 @@
       </service>
     </group>
 
-    <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
-    <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
-      <direction>UPGRADE</direction>
-      <skippable>true</skippable>
-      <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>upgrade_oozie_database_and_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
-    <!-- Only create the ShareLib folder during a Downgrade. -->
-    <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
-      <direction>DOWNGRADE</direction>
-      <skippable>true</skippable>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>create_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <group xsi:type="restart" name="OOZIE" title="Oozie">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -774,14 +749,14 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
index 816fe13..df12920 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml
@@ -689,31 +689,6 @@
       </service>
     </group>
 
-    <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
-    <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
-      <direction>UPGRADE</direction>
-      <skippable>true</skippable>
-      <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>upgrade_oozie_database_and_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
-    <!-- Only create the ShareLib folder during a Downgrade. -->
-    <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
-      <direction>DOWNGRADE</direction>
-      <skippable>true</skippable>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>create_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <group xsi:type="restart" name="OOZIE" title="Oozie">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -1062,14 +1037,14 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index 5166c4b..ff7e032 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -753,31 +753,6 @@
       </service>
     </group>
 
-    <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
-    <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
-      <direction>UPGRADE</direction>
-      <skippable>true</skippable>
-      <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>upgrade_oozie_database_and_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
-    <!-- Only create the ShareLib folder during a Downgrade. -->
-    <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
-      <direction>DOWNGRADE</direction>
-      <skippable>true</skippable>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>create_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <group xsi:type="restart" name="OOZIE" title="Oozie">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -1153,14 +1128,14 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index 8b2f2a3..69dbe58 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -594,24 +594,24 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index 375da12..7e7138d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -741,24 +741,24 @@
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
 
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index ea3d6a6..34e4848 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -753,24 +753,24 @@
           <task xsi:type="configure" id="hdp_2_3_0_0_oozie_remove_redundant_configurations"/>
           <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
 
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
index 34423f9..57b2f42 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml
@@ -473,31 +473,6 @@
       </service>
     </group>
 
-    <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
-    <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
-      <direction>UPGRADE</direction>
-      <skippable>true</skippable>
-      <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>upgrade_oozie_database_and_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
-    <!-- Only create the ShareLib folder during a Downgrade. -->
-    <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
-      <direction>DOWNGRADE</direction>
-      <skippable>true</skippable>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>create_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <group xsi:type="restart" name="OOZIE" title="Oozie">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -879,14 +854,14 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index 50c8584..f58eb79 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -492,31 +492,6 @@
       </service>
     </group>
 
-    <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
-    <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
-      <direction>UPGRADE</direction>
-      <skippable>true</skippable>
-      <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>upgrade_oozie_database_and_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
-    <!-- Only create the ShareLib folder during a Downgrade. -->
-    <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
-      <direction>DOWNGRADE</direction>
-      <skippable>true</skippable>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>create_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <group xsi:type="restart" name="OOZIE" title="Oozie">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -898,14 +873,14 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
index 876342c..ab955e4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
@@ -689,24 +689,24 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index f145de1..573713c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -644,24 +644,24 @@
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_4_0_0_oozie_remove_service_classes" />
         
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml
index a9ee860..2600eae 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml
@@ -474,31 +474,6 @@
       </service>
     </group>
 
-    <!-- Upgrade Oozie DB only on Upgrade direction, and always create a new ShareLib. -->
-    <group name="UPGRADE_OOZIE" title="Upgrade Oozie Database">
-      <direction>UPGRADE</direction>
-      <skippable>true</skippable>
-      <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>upgrade_oozie_database_and_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
-    <!-- Only create the ShareLib folder during a Downgrade. -->
-    <group name="DOWNGRADE_OOZIE" title="Downgrade Oozie ShareLib">
-      <direction>DOWNGRADE</direction>
-      <skippable>true</skippable>
-      <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
-          <script>scripts/oozie_server_upgrade.py</script>
-          <function>create_sharelib</function>
-        </task>
-      </execute-stage>
-    </group>
-
     <group xsi:type="restart" name="OOZIE" title="Oozie">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -880,14 +855,14 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
index b8bd1ee..b91e27a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.4.xml
@@ -687,24 +687,24 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>
         </pre-upgrade>
 
         <pre-downgrade>
-          <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
             <script>scripts/oozie_server.py</script>
             <function>stop</function>
           </task>
 
-          <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Create a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>create_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index 7e61942..eb5bf62 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -54,6 +54,8 @@ import org.apache.ambari.server.state.stack.UpgradePack;
 import org.apache.ambari.server.state.stack.upgrade.ConfigUpgradeChangeDefinition;
 import org.apache.ambari.server.state.stack.upgrade.ConfigureTask;
 import org.apache.ambari.server.state.stack.upgrade.Direction;
+import org.apache.ambari.server.state.stack.upgrade.ExecuteTask;
+import org.apache.ambari.server.state.stack.upgrade.Grouping;
 import org.apache.ambari.server.state.stack.upgrade.ManualTask;
 import org.apache.ambari.server.state.stack.upgrade.StageWrapper;
 import org.apache.ambari.server.state.stack.upgrade.Task;
@@ -202,13 +204,14 @@ public class UpgradeHelperTest {
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
 
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
     assertEquals("PRE_CLUSTER", groups.get(0).name);
     assertEquals("ZOOKEEPER", groups.get(1).name);
     assertEquals("CORE_MASTER", groups.get(2).name);
     assertEquals("CORE_SLAVES", groups.get(3).name);
     assertEquals("HIVE", groups.get(4).name);
+    assertEquals("OOZIE", groups.get(5).name);
 
     UpgradeGroupHolder holder = groups.get(2);
     boolean found = false;
@@ -238,7 +241,7 @@ public class UpgradeHelperTest {
     assertTrue(mt.structuredOut.contains("DATANODE"));
     assertTrue(mt.structuredOut.contains("NODEMANAGER"));
 
-    UpgradeGroupHolder postGroup = groups.get(5);
+    UpgradeGroupHolder postGroup = groups.get(6);
     assertEquals("POST_CLUSTER", postGroup.name);
     assertEquals("Finalize Upgrade", postGroup.title);
     assertEquals(3, postGroup.items.size());
@@ -249,7 +252,7 @@ public class UpgradeHelperTest {
 
     assertEquals(4, groups.get(0).items.size());
     assertEquals(6, groups.get(1).items.size());
-    assertEquals(8, groups.get(2).items.size());
+    assertEquals(9, groups.get(2).items.size());
     assertEquals(8, groups.get(3).items.size());
   }
 
@@ -321,7 +324,7 @@ public class UpgradeHelperTest {
         UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING);
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
     for (UpgradeGroupHolder group : groups) {
       for (StageWrapper stageWrapper : group.items) {
@@ -349,7 +352,7 @@ public class UpgradeHelperTest {
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
 
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
     UpgradeGroupHolder mastersGroup = groups.get(2);
     assertEquals("CORE_MASTER", mastersGroup.name);
@@ -400,14 +403,16 @@ public class UpgradeHelperTest {
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
 
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
     assertEquals("PRE_CLUSTER", groups.get(0).name);
     assertEquals("ZOOKEEPER", groups.get(1).name);
     assertEquals("CORE_MASTER", groups.get(2).name);
     assertEquals("CORE_SLAVES", groups.get(3).name);
+    assertEquals("HIVE", groups.get(4).name);
+    assertEquals("OOZIE", groups.get(5).name);
 
-    UpgradeGroupHolder postGroup = groups.get(5);
+    UpgradeGroupHolder postGroup = groups.get(6);
     assertEquals("POST_CLUSTER", postGroup.name);
     assertEquals("Finalize Upgrade", postGroup.title);
     assertEquals(3, postGroup.items.size());
@@ -417,7 +422,7 @@ public class UpgradeHelperTest {
     assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType());
 
     assertEquals(6, groups.get(1).items.size());
-    assertEquals(8, groups.get(2).items.size());
+    assertEquals(9, groups.get(2).items.size());
     assertEquals(7, groups.get(3).items.size());
   }
 
@@ -435,17 +440,17 @@ public class UpgradeHelperTest {
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
 
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
-    UpgradeGroupHolder preGroup = groups.get(0);
-    assertEquals("PRE_CLUSTER", preGroup.name);
-    assertEquals("HIVE", groups.get(1).name);
-    assertEquals("CORE_SLAVES", groups.get(2).name);
-    assertEquals("CORE_MASTER", groups.get(3).name);
-    assertEquals("ZOOKEEPER", groups.get(4).name);
+    assertEquals("PRE_CLUSTER", groups.get(0).name);
+    assertEquals("OOZIE", groups.get(1).name);
+    assertEquals("HIVE", groups.get(2).name);
+    assertEquals("CORE_SLAVES", groups.get(3).name);
+    assertEquals("CORE_MASTER", groups.get(4).name);
+    assertEquals("ZOOKEEPER", groups.get(5).name);
 
 
-    UpgradeGroupHolder postGroup = groups.get(5);
+    UpgradeGroupHolder postGroup = groups.get(6);
     assertEquals("POST_CLUSTER", postGroup.name);
     assertEquals("Finalize Downgrade", postGroup.title);
     assertEquals(3, postGroup.items.size());
@@ -455,10 +460,10 @@ public class UpgradeHelperTest {
     assertEquals(StageWrapper.Type.SERVER_SIDE_ACTION, postGroup.items.get(2).getType());
 
     assertEquals(4, groups.get(0).items.size());
-    assertEquals(2, groups.get(1).items.size());
-    assertEquals(8, groups.get(2).items.size());
-    assertEquals(7, groups.get(3).items.size());
-    assertEquals(5, groups.get(4).items.size());
+    assertEquals(8, groups.get(1).items.size());
+    assertEquals(3, groups.get(2).items.size());
+    assertEquals(8, groups.get(3).items.size());
+    assertEquals(8, groups.get(4).items.size());
   }
 
   @Test
@@ -478,7 +483,17 @@ public class UpgradeHelperTest {
     assertEquals(1, groups.size());
     UpgradeGroupHolder group = groups.iterator().next();
 
-    assertEquals(6, group.items.size());
+    // Pre:
+    //   Manual task = 1
+    //   2x - Execute task on all 3 = 6
+
+    // Post:
+    //   Execute task on all 3 = 3
+    //   2x - Manual task = 2
+    //   3x - Execute task on all 3 = 9
+
+    // Service Check = 1
+    assertEquals(22, group.items.size());
   }
 
   @Test
@@ -495,7 +510,7 @@ public class UpgradeHelperTest {
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
 
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
     // grab the manual task out of ZK which has placeholder text
     UpgradeGroupHolder zookeeperGroup = groups.get(1);
@@ -523,13 +538,12 @@ public class UpgradeHelperTest {
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
 
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
     // grab the configure task out of Hive
     UpgradeGroupHolder hiveGroup = groups.get(4);
     assertEquals("HIVE", hiveGroup.name);
-    ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(
-        1).getTasks().get(0);
+    ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0);
 
     // now change the thrift port to http to have the 2nd condition invoked
     Map<String, String> hiveConfigs = new HashMap<String, String>();
@@ -581,10 +595,8 @@ public class UpgradeHelperTest {
 
     assertEquals("delete-if-key-present", transfers.get(8).deleteKey);
     assertEquals("delete-if-key-absent", transfers.get(9).deleteKey);
-
   }
 
-
   @Test
   public void testConfigureTask() throws Exception {
     Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
@@ -601,7 +613,7 @@ public class UpgradeHelperTest {
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade,
         context);
 
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
     // grab the configure task out of Hive
     UpgradeGroupHolder hiveGroup = groups.get(4);
@@ -674,12 +686,12 @@ public class UpgradeHelperTest {
 
     List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
 
-    assertEquals(6, groups.size());
+    assertEquals(7, groups.size());
 
     // grab the configure task out of Hive
     UpgradeGroupHolder hiveGroup = groups.get(4);
     assertEquals("HIVE", hiveGroup.name);
-    ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(1).getTasks().get(1).getTasks().get(0);
+    ConfigureTask configureTask = (ConfigureTask) hiveGroup.items.get(2).getTasks().get(0).getTasks().get(0);
 
     Map<String, String> configProperties = configureTask.getConfigurationChanges(cluster, cup);
     assertFalse(configProperties.isEmpty());
@@ -754,7 +766,7 @@ public class UpgradeHelperTest {
 
     UpgradeGroupHolder holder = groups.get(3);
     assertEquals(holder.name, "SERVICE_CHECK_1");
-    assertEquals(6, holder.items.size());
+    assertEquals(7, holder.items.size());
     int numServiceChecksActual = 0;
     for (StageWrapper sw : holder.items) {
       for(Service service : services) {
@@ -881,7 +893,7 @@ public class UpgradeHelperTest {
 
     assertEquals(4, groups.get(0).items.size());
     assertEquals(5, groups.get(1).items.size());
-    assertEquals(8, groups.get(2).items.size());
+    assertEquals(9, groups.get(2).items.size());
     assertEquals(8, groups.get(3).items.size());
   }
 
@@ -934,6 +946,7 @@ public class UpgradeHelperTest {
     c.addService(serviceFactory.createNew(c, "YARN"));
     c.addService(serviceFactory.createNew(c, "ZOOKEEPER"));
     c.addService(serviceFactory.createNew(c, "HIVE"));
+    c.addService(serviceFactory.createNew(c, "OOZIE"));
 
     Service s = c.getService("HDFS");
     ServiceComponent sc = s.addServiceComponent("NAMENODE");
@@ -962,6 +975,16 @@ public class UpgradeHelperTest {
     sc = s.addServiceComponent("HIVE_SERVER");
     sc.addServiceComponentHost("h2");
 
+    s = c.getService("OOZIE");
+    // Oozie Server HA
+    sc = s.addServiceComponent("OOZIE_SERVER");
+    sc.addServiceComponentHost("h2");
+    sc.addServiceComponentHost("h3");
+    sc = s.addServiceComponent("OOZIE_CLIENT");
+    sc.addServiceComponentHost("h1");
+    sc.addServiceComponentHost("h2");
+    sc.addServiceComponentHost("h3");
+
     // set some desired configs
     Map<String, String> hiveConfigs = new HashMap<String, String>();
     hiveConfigs.put("hive.server2.transport.mode", "binary");
@@ -1007,12 +1030,23 @@ public class UpgradeHelperTest {
     expect(m_masterHostResolver.getMasterAndHosts("YARN", "RESOURCEMANAGER")).andReturn(type).anyTimes();
 
     type = new HostsType();
+    expect(m_masterHostResolver.getMasterAndHosts("YARN", "APP_TIMELINE_SERVER")).andReturn(type).anyTimes();
+
+    type = new HostsType();
     type.hosts.addAll(Arrays.asList("h1", "h3"));
     expect(m_masterHostResolver.getMasterAndHosts("YARN", "NODEMANAGER")).andReturn(type).anyTimes();
 
     expect(m_masterHostResolver.getMasterAndHosts("HIVE", "HIVE_SERVER")).andReturn(
         type).anyTimes();
 
+    type = new HostsType();
+    type.hosts.addAll(Arrays.asList("h2", "h3"));
+    expect(m_masterHostResolver.getMasterAndHosts("OOZIE", "OOZIE_SERVER")).andReturn(type).anyTimes();
+
+    type = new HostsType();
+    type.hosts.addAll(Arrays.asList("h1", "h2", "h3"));
+    expect(m_masterHostResolver.getMasterAndHosts("OOZIE", "OOZIE_CLIENT")).andReturn(type).anyTimes();
+
     expect(m_masterHostResolver.getCluster()).andReturn(c).anyTimes();
 
     replay(m_masterHostResolver);
@@ -1020,6 +1054,76 @@ public class UpgradeHelperTest {
     return c;
   }
 
+  /**
+   * Test that multiple execute tasks with an annotation of synchronized="true" each run in their own stage.
+   */
+  @Test
+  public void testUpgradeWithMultipleTasksInOwnStage() throws Exception {
+    Map<String, UpgradePack> upgrades = ambariMetaInfo.getUpgradePacks("HDP", "2.1.1");
+
+    assertTrue(upgrades.containsKey("upgrade_test"));
+    UpgradePack upgrade = upgrades.get("upgrade_test");
+    assertNotNull(upgrade);
+    assertTrue(upgrade.getType() == UpgradeType.ROLLING);
+
+    List<Grouping> upgradePackGroups = upgrade.getGroups(Direction.UPGRADE);
+
+    boolean foundService = false;
+    for (Grouping group : upgradePackGroups) {
+      if (group.title.equals("Oozie")) {
+        foundService = true;
+      }
+    }
+    assertTrue(foundService);
+
+    makeCluster();
+
+    UpgradeContext context = new UpgradeContext(m_masterHostResolver, HDP_21,
+        HDP_21, UPGRADE_VERSION, Direction.UPGRADE, UpgradeType.ROLLING);
+
+    List<UpgradeGroupHolder> groups = m_upgradeHelper.createSequence(upgrade, context);
+
+    // The upgrade pack has 2 tasks for Oozie in the pre-upgrade group.
+    // The first task runs on "all", i.e., both Oozie Servers, whereas the
+    // second task runs on "any", i.e., exactly one.
+    int numPrepareStages = 0;
+    for (UpgradeGroupHolder group : groups) {
+      if (group.name.equals("OOZIE")) {
+        assertTrue(group.items.size() > 0);
+        for (StageWrapper sw : group.items) {
+
+          if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2 (Batch 1 of 2)") ||
+              sw.getText().equalsIgnoreCase("Preparing Oozie Server on h3 (Batch 2 of 2)")) {
+            numPrepareStages++;
+            List<TaskWrapper> taskWrappers = sw.getTasks();
+            assertEquals(1, taskWrappers.size());
+            List<Task> tasks = taskWrappers.get(0).getTasks();
+            assertEquals(1, taskWrappers.get(0).getHosts().size());
+            assertEquals(1, tasks.size());
+
+            ExecuteTask task = (ExecuteTask) tasks.get(0);
+            assertTrue("scripts/oozie_server.py".equalsIgnoreCase(task.script));
+            assertTrue("stop".equalsIgnoreCase(task.function));
+          }
+
+          if (sw.getText().equalsIgnoreCase("Preparing Oozie Server on h2")) {
+            numPrepareStages++;
+            List<TaskWrapper> taskWrappers = sw.getTasks();
+            assertEquals(1, taskWrappers.size());
+            List<Task> tasks = taskWrappers.get(0).getTasks();
+            assertEquals(1, taskWrappers.get(0).getHosts().size());
+            assertEquals(1, tasks.size());
+
+            ExecuteTask task = (ExecuteTask) tasks.get(0);
+            assertTrue("scripts/oozie_server_upgrade.py".equalsIgnoreCase(task.script));
+            assertTrue("upgrade_oozie_database_and_sharelib".equalsIgnoreCase(task.function));
+          }
+        }
+      }
+    }
+    assertEquals(3, numPrepareStages);
+  }
+
   @Test
   public void testDowngradeAfterPartialUpgrade() throws Exception {
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml
new file mode 100644
index 0000000..6010659
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/HDP/2.0.7/services/OOZIE/metainfo.xml
@@ -0,0 +1,176 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>OOZIE</name>
+      <displayName>Oozie</displayName>
+      <comment>System for workflow coordination and execution of Apache Hadoop jobs.  This also includes the installation of the optional Oozie Web Console which relies on and will install the &lt;a target="_blank" href="http://www.sencha.com/legal/open-source-faq/"&gt;ExtJS&lt;/a&gt; Library.</comment>
+      <version>4.2.0.2.3</version>
+
+      <components>
+        <component>
+          <name>OOZIE_SERVER</name>
+          <displayName>Oozie Server</displayName>
+          <category>MASTER</category>
+          <cardinality>1+</cardinality>
+          <versionAdvertised>true</versionAdvertised>
+          <dependencies>
+            <dependency>
+              <name>HDFS/HDFS_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+            <dependency>
+              <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+            <dependency>
+              <name>YARN/YARN_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
+          <commandScript>
+            <script>scripts/oozie_server.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>1200</timeout>
+          </commandScript>
+        </component>
+
+        <component>
+          <name>OOZIE_CLIENT</name>
+          <displayName>Oozie Client</displayName>
+          <category>CLIENT</category>
+          <cardinality>1+</cardinality>
+          <versionAdvertised>true</versionAdvertised>
+          <dependencies>
+            <dependency>
+              <name>HDFS/HDFS_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+            <dependency>
+              <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
+          <commandScript>
+            <script>scripts/oozie_client.py</script>
+            <scriptType>PYTHON</scriptType>
+          </commandScript>
+          <configFiles>
+            <configFile>
+              <type>xml</type>
+              <fileName>oozie-site.xml</fileName>
+              <dictionaryName>oozie-site</dictionaryName>
+            </configFile>
+            <configFile>
+              <type>env</type>
+              <fileName>oozie-env.sh</fileName>
+              <dictionaryName>oozie-env</dictionaryName>
+            </configFile>
+            <configFile>
+              <type>env</type>
+              <fileName>oozie-log4j.properties</fileName>
+              <dictionaryName>oozie-log4j</dictionaryName>
+            </configFile>
+          </configFiles>
+        </component>
+      </components>
+
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+
+      <requiredServices>
+        <service>YARN</service>
+      </requiredServices>
+
+      <configuration-dependencies>
+        <config-type>oozie-site</config-type>
+        <config-type>oozie-env</config-type>
+        <config-type>oozie-log4j</config-type>
+        <config-type>yarn-site</config-type>
+        <config-type>hive-site</config-type>
+        <config-type>tez-site</config-type>
+      </configuration-dependencies>
+
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>zip</name>
+            </package>
+            <package>
+              <name>mysql-connector-java</name>
+              <skipUpgrade>true</skipUpgrade>
+            </package>
+            <package>
+              <name>extjs</name>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>redhat5,redhat6,redhat7,suse11</osFamily>
+          <packages>
+            <package>
+              <name>oozie_2_3_*</name>
+            </package>
+            <package>
+              <name>falcon_2_3_*</name>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
+          <packages>
+            <package>
+              <name>oozie-2-3-.*</name>
+            </package>
+            <package>
+              <name>falcon-2-3-.*</name>
+            </package>
+            <package>
+              <name>extjs</name>
+            </package>
+            <package>
+              <name>libxml2-utils</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
+    </service>
+  </services>
+</metainfo>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml
index 0e6d914..085ba25 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_bucket_test.xml
@@ -46,36 +46,34 @@
       <component name="ZOOKEEPER_SERVER">
         <pre-upgrade>
           <task xsi:type="manual">
-            <message>this is pre</message>
+            <message>pre-manual 1 task to run on all hosts</message>
           </task>
           <task xsi:type="execute">
-            <command>ls</command>
+            <command>pre-ls 1 to run on all hosts</command>
           </task>
           <task xsi:type="execute">
-            <command>ls</command>
+            <command>pre-ls 2 to run on all hosts</command>
           </task>
-          
         </pre-upgrade>
         
         <post-upgrade>
           <task xsi:type="execute">
-            <command>ls</command>
+            <command>post-ls 1 to run on all hosts</command>
           </task>
           <task xsi:type="manual">
-            <message>this is pre</message>
+            <message>post-manual 1 task to run on all hosts</message>
           </task>
           <task xsi:type="manual">
-            <message>this is pre</message>
+            <message>post-manual 2 task to run on all hosts</message>
           </task>
           <task xsi:type="execute">
-            <command>ls</command>
+            <command>post-ls 2 to run on all hosts</command>
           </task>
           <task xsi:type="execute">
-            <command>ls</command>
+            <command>post-ls 3 to run on all hosts</command>
           </task>
-
           <task xsi:type="execute">
-            <command>ls</command>
+            <command>post-ls 4 to run on all hosts</command>
           </task>
           
         </post-upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml
index 8f78d0f..c79c8e7 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_nonrolling_new_stack.xml
@@ -442,7 +442,7 @@
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
       <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Upgrade Oozie Database">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+        <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib">
           <script>scripts/oozie_server_upgrade.py</script>
           <function>upgrade_oozie_database_and_sharelib</function>
         </task>
@@ -454,7 +454,7 @@
       <direction>DOWNGRADE</direction>
       <skippable>true</skippable>
       <execute-stage service="OOZIE" component="OOZIE_SERVER" title="Downgrade Oozie ShareLib">
-        <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+        <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib">
           <script>scripts/oozie_server_upgrade.py</script>
           <function>create_sharelib</function>
         </task>
@@ -809,7 +809,7 @@
     <service name="OOZIE">
       <component name="OOZIE_SERVER">
         <pre-upgrade>
-          <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+          <task xsi:type="execute" hosts="any" summary="Upgrading the Oozie database and creating a new sharelib">
             <script>scripts/oozie_server_upgrade.py</script>
             <function>upgrade_oozie_database_and_sharelib</function>
           </task>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cbed6402/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
index 4f71b4a..237da77 100644
--- a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_test.xml
@@ -101,7 +101,17 @@
         <component>HIVE_SERVER</component>
         <component>WEBHCAT_SERVER</component>
       </service>
-    </group>    
+    </group>
+
+    <group name="OOZIE" title="Oozie">
+      <skippable>true</skippable>
+      <supports-auto-skip-failure>false</supports-auto-skip-failure>
+      <service-check>false</service-check>
+      <service name="OOZIE">
+        <component>OOZIE_SERVER</component>
+        <component>OOZIE_CLIENT</component>
+      </service>
+    </group>
     
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
       <execute-stage title="Confirm Finalize">
@@ -199,13 +209,38 @@
             <summary>HiveServer Port Availability</summary>
             <message>The HiveServer port will now change to 10010 if hive is using a binary transfer mode or 10011 if hive is using an http transport mode. You can use "netstat -anp | grep 1001[01]" to determine if the port is available on each of following HiveServer host(s): {{hosts.all}}. If the port is not available, the process using it must be terminated.</message>
           </task>
-
           <task xsi:type="configure" id="hdp_2_1_1_set_transport_mode"/>
-
           <task xsi:type="configure" id="hdp_2_1_1_hive_server_foo"/>
-
         </pre-upgrade>
        </component>
-     </service>    
+     </service>
+
+    <service name="OOZIE">
+      <component name="OOZIE_SERVER">
+        <pre-upgrade>
+          <!-- This is important, do not remove it since UpgradeHelperTest.java :
+          testUpgradeWithMultipleTasksWithMultipleHostTypes() asserts
+          that these tasks each run on their own stage. -->
+          <task xsi:type="execute" hosts="all" sequential="true" summary="Shut down all Oozie servers">
+            <script>scripts/oozie_server.py</script>
+            <function>stop</function>
+          </task>
+
+          <task xsi:type="execute" hosts="any" sequential="true" summary="Upgrading the Oozie database and creating a new sharelib">
+            <script>scripts/oozie_server_upgrade.py</script>
+            <function>upgrade_oozie_database_and_sharelib</function>
+          </task>
+        </pre-upgrade>
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+
+      <component name="OOZIE_CLIENT">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+    </service>
   </processing>
 </upgrade>


[05/19] ambari git commit: AMBARI-14665. Race condition problem in ServiceComponentHostImpl when provisioning cluster with Blueprint (Sandor Magyari via rlevas)

Posted by nc...@apache.org.
AMBARI-14665. Race condition problem in ServiceComponentHostImpl when provisioning cluster with Blueprint (Sandor Magyari via rlevas)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0cbefb3e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0cbefb3e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0cbefb3e

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 0cbefb3e6ceebea32d71461152aa1f141a8ed5d6
Parents: 36af9bb
Author: Sandor Magyari <sm...@hortonworks.com>
Authored: Mon Jan 18 13:35:32 2016 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Mon Jan 18 13:36:02 2016 -0500

----------------------------------------------------------------------
 .../svccomphost/ServiceComponentHostImpl.java   | 65 +++++++++++++++-----
 1 file changed, 49 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/0cbefb3e/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
index 7bc4680..bfb6214 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/svccomphost/ServiceComponentHostImpl.java
@@ -821,7 +821,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
       HostComponentStateEntity stateEntity = getStateEntity();
       if (stateEntity != null) {
         getStateEntity().setCurrentState(state);
-        saveIfPersisted();
+        saveComponentStateEntityIfPersisted();
       } else {
         LOG.warn("Setting a member on an entity object that may have been " +
           "previously deleted, serviceName = " + getServiceName() + ", " +
@@ -860,7 +860,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
       HostComponentStateEntity stateEntity = getStateEntity();
       if (stateEntity != null) {
         getStateEntity().setVersion(version);
-        saveIfPersisted();
+        saveComponentStateEntityIfPersisted();
       } else {
         LOG.warn("Setting a member on an entity object that may have been " +
           "previously deleted, serviceName = " + getServiceName() + ", " +
@@ -899,7 +899,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
       HostComponentStateEntity stateEntity = getStateEntity();
       if (stateEntity != null) {
         getStateEntity().setSecurityState(securityState);
-        saveIfPersisted();
+        saveComponentStateEntityIfPersisted();
       } else {
         LOG.warn("Setting a member on an entity object that may have been " +
           "previously deleted, serviceName = " + getServiceName() + ", " +
@@ -940,8 +940,11 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
 
     writeLock.lock();
     try {
+      LOG.debug("Set DesiredSecurityState on serviceName = {} componentName = {} hostName = {} to {}",
+        getServiceName(), getServiceComponentName(), getHostName(), securityState);
+
       getDesiredStateEntity().setSecurityState(securityState);
-      saveIfPersisted();
+      saveComponentDesiredStateEntityIfPersisted();
     } finally {
       writeLock.unlock();
     }
@@ -963,7 +966,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
       HostComponentStateEntity stateEntity = getStateEntity();
       if (stateEntity != null) {
         stateEntity.setUpgradeState(upgradeState);
-        saveIfPersisted();
+        saveComponentStateEntityIfPersisted();
       } else {
         LOG.warn("Setting a member on an entity object that may have been " +
           "previously deleted, serviceName = " + getServiceName() + ", " +
@@ -1015,7 +1018,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
         try {
           stateMachine.doTransition(event.getType(), event);
           getStateEntity().setCurrentState(stateMachine.getCurrentState());
-          saveIfPersisted();
+          saveComponentStateEntityIfPersisted();
           // TODO Audit logs
         } catch (InvalidStateTransitionException e) {
           LOG.error("Can't handle ServiceComponentHostEvent event at"
@@ -1175,7 +1178,7 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
       HostComponentStateEntity stateEntity = getStateEntity();
       if (stateEntity != null) {
         stateEntity.setCurrentStack(stackEntity);
-        saveIfPersisted();
+        saveComponentStateEntityIfPersisted();
       } else {
         LOG.warn("Setting a member on an entity object that may have been " +
           "previously deleted, serviceName = " + getServiceName() + ", " +
@@ -1212,10 +1215,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
   public void setDesiredState(State state) {
     writeLock.lock();
     try {
+      LOG.debug("Set DesiredState on serviceName = {} componentName = {} hostName = {} to {} ",
+        getServiceName(), getServiceComponentName(), getHostName(), state);
+
       HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
       if (desiredStateEntity != null) {
         desiredStateEntity.setDesiredState(state);
-        saveIfPersisted();
+        saveComponentDesiredStateEntityIfPersisted();
       } else {
         LOG.warn("Setting a member on an entity object that may have been " +
           "previously deleted, serviceName = " + getServiceName() + ", " +
@@ -1253,13 +1259,16 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
   public void setDesiredStackVersion(StackId stackId) {
     writeLock.lock();
     try {
+      LOG.debug("Set DesiredStackVersion on serviceName = {} componentName = {} hostName = {} to {}",
+        getServiceName(), getServiceComponentName(), getHostName(), stackId);
+
       HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
       if (desiredStateEntity != null) {
         StackEntity stackEntity = stackDAO.find(stackId.getStackName(),
           stackId.getStackVersion());
 
         desiredStateEntity.setDesiredStack(stackEntity);
-        saveIfPersisted();
+        saveComponentDesiredStateEntityIfPersisted();
       }
     } finally {
       writeLock.unlock();
@@ -1290,10 +1299,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
   public void setComponentAdminState(HostComponentAdminState attribute) {
     writeLock.lock();
     try {
+      LOG.debug("Set ComponentAdminState on serviceName = {} componentName = {} hostName = {} to {}",
+        getServiceName(), getServiceComponentName(), getHostName(), attribute);
+
       HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
       if (desiredStateEntity != null) {
         desiredStateEntity.setAdminState(attribute);
-        saveIfPersisted();
+        saveComponentDesiredStateEntityIfPersisted();
       } else {
         LOG.warn("Setting a member on an entity object that may have been " +
           "previously deleted, serviceName = " + getServiceName() + ", " +
@@ -1426,7 +1438,8 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
 
           eventPublisher.publish(event);
         } else {
-          saveIfPersisted();
+          saveComponentStateEntityIfPersisted();
+          saveComponentDesiredStateEntityIfPersisted();
         }
       } finally {
         writeLock.unlock();
@@ -1481,19 +1494,33 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
   }
 
   /**
-   * Merges the encapsulated {@link HostComponentStateEntity} and
-   * {@link HostComponentDesiredStateEntity} inside of a new transaction. This
+   * Merges the encapsulated {@link HostComponentStateEntity} inside of a new transaction. This
    * method assumes that the appropriate write lock has already been acquired
    * from {@link #readWriteLock}.
    */
   @Transactional
-  void saveIfPersisted() {
+  void saveComponentStateEntityIfPersisted() {
     if (isPersisted()) {
       hostComponentStateDAO.merge(stateEntity);
+    }
+  }
+
+  /**
+   * Merges the encapsulated {@link HostComponentDesiredStateEntity} inside of a new transaction. This
+   * method assumes that the appropriate write lock has already been acquired
+   * from {@link #readWriteLock}.
+   */
+  @Transactional
+  void saveComponentDesiredStateEntityIfPersisted() {
+    if (isPersisted()) {
+      LOG.debug("Save desiredStateEntity serviceName = {} componentName = {} hostName = {} desiredState = {}",
+        getServiceName(), getServiceComponentName(), getHostName(), desiredStateEntity.getDesiredState());
+
       hostComponentDesiredStateDAO.merge(desiredStateEntity);
     }
   }
 
+
   @Override
   public boolean canBeRemoved() {
     clusterGlobalLock.readLock().lock();
@@ -1645,10 +1672,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
   public void setMaintenanceState(MaintenanceState state) {
     writeLock.lock();
     try {
+      LOG.debug("Set MaintenanceState on serviceName = {} componentName = {} hostName = {} to {}",
+        getServiceName(), getServiceComponentName(), getHostName(), state);
+
       HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
       if (desiredStateEntity != null) {
         desiredStateEntity.setMaintenanceState(state);
-        saveIfPersisted();
+        saveComponentDesiredStateEntityIfPersisted();
 
         // broadcast the maintenance mode change
         MaintenanceModeEvent event = new MaintenanceModeEvent(state, this);
@@ -1708,10 +1738,13 @@ public class ServiceComponentHostImpl implements ServiceComponentHost {
   public void setRestartRequired(boolean restartRequired) {
     writeLock.lock();
     try {
+      LOG.debug("Set RestartRequired on serviceName = {} componentName = {} hostName = {} to {}",
+        getServiceName(), getServiceComponentName(), getHostName(), restartRequired);
+
       HostComponentDesiredStateEntity desiredStateEntity = getDesiredStateEntity();
       if (desiredStateEntity != null) {
         desiredStateEntity.setRestartRequired(restartRequired);
-        saveIfPersisted();
+        saveComponentDesiredStateEntityIfPersisted();
         helper.invalidateStaleConfigsCache(this);
       } else {
         LOG.warn("Setting a member on an entity object that may have been " +


[09/19] ambari git commit: AMBARI-14698. Kerberos automation logic should use stack advisor when determining configuration updates (rlevas)

Posted by nc...@apache.org.
AMBARI-14698. Kerberos automation logic should use stack advisor when determining configuration updates (rlevas)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f3ffe0c2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f3ffe0c2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f3ffe0c2

Branch: refs/heads/branch-dev-patch-upgrade
Commit: f3ffe0c2094bd619cffbdd2daf4a0aa7e033f128
Parents: 132c535
Author: Robert Levas <rl...@hortonworks.com>
Authored: Mon Jan 18 20:01:29 2016 -0500
Committer: Robert Levas <rl...@hortonworks.com>
Committed: Mon Jan 18 20:01:29 2016 -0500

----------------------------------------------------------------------
 .../server/controller/KerberosHelper.java       |  41 +++-
 .../server/controller/KerberosHelperImpl.java   | 210 +++++++++++++++++--
 .../BlueprintConfigurationProcessor.java        |   5 -
 .../serveraction/ServerActionExecutor.java      |   8 +
 .../AbstractPrepareKerberosServerAction.java    |  90 +++++---
 .../PrepareDisableKerberosServerAction.java     |   2 +-
 .../PrepareEnableKerberosServerAction.java      |   2 +-
 .../PrepareKerberosIdentitiesServerAction.java  |   3 +-
 .../topology/ClusterConfigurationRequest.java   |  17 +-
 .../stacks/HDP/2.0.6/services/stack_advisor.py  |   2 +-
 .../StackAdvisorBlueprintProcessorTest.java     |   3 +
 .../server/controller/KerberosHelperTest.java   |  46 +++-
 .../ClusterConfigurationRequestTest.java        |   3 +-
 13 files changed, 362 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
index b6686b4..5008d60 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelper.java
@@ -224,6 +224,8 @@ public interface KerberosHelper {
    * @param cluster                the cluster
    * @param existingConfigurations the cluster's existing configurations
    * @param services               the set of services to process
+   * @param kerberosEnabled        true if kerberos is (to be) enabled; otherwise false
+   * @param applyStackAdvisorUpdates true to invoke the stack advisor to validate property updates; false to skip
    * @return a map of configuration updates
    * @throws AmbariException
    * @throws KerberosInvalidConfigurationException if an issue occurs trying to get the
@@ -231,13 +233,37 @@ public interface KerberosHelper {
    */
   Map<String, Map<String, String>> getServiceConfigurationUpdates(Cluster cluster,
                                                                   Map<String, Map<String, String>> existingConfigurations,
-                                                                  Set<String> services, boolean serviceAlreadyExists)
+                                                                  Set<String> services,
+                                                                  boolean serviceAlreadyExists,
+                                                                  boolean kerberosEnabled,
+                                                                  boolean applyStackAdvisorUpdates)
       throws KerberosInvalidConfigurationException, AmbariException;
 
   /**
-   * Ensures that the relevant headless (or user) Kerberos identities are created and cached.
+   * Invokes the Stack Advisor to help determine relevant configuration changes when enabling or
+   * disabling Kerberos
    *
+   * @param cluster                a cluster
+   * @param services               a set of services that are being configured to enabled or disable Kerberos
+   * @param existingConfigurations the cluster's existing configurations
+   * @param kerberosConfigurations the configuration updates to make (must not be mutable)
+   * @param propertiesToIgnore     the configuration properties that should be ignored when applying recommendations
+   * @param kerberosEnabled        true if kerberos is (to be) enabled; otherwise false
+   * @return the configuration updates
+   * @throws AmbariException
+   */
+  Map<String, Map<String, String>> applyStackAdvisorUpdates(Cluster cluster, Set<String> services,
+                                                            Map<String, Map<String, String>> existingConfigurations,
+                                                            Map<String, Map<String, String>> kerberosConfigurations,
+                                                            Map<String, Set<String>> propertiesToIgnore,
+                                                            boolean kerberosEnabled)
+      throws AmbariException;
+
+  /**
+   * Ensures that the relevant headless (or user) Kerberos identities are created and cached.
+   * <p/>
    * This can be called any number of times and only the missing identities will be created.
+   *
    * @param cluster                the cluster
    * @param existingConfigurations the cluster's existing configurations
    * @param services               the set of services to process
@@ -453,6 +479,17 @@ public interface KerberosHelper {
   Boolean getManageIdentitiesDirective(Map<String, String> requestProperties);
 
   /**
+   * Given a list of KerberosIdentityDescriptors, returns a Map fo configuration types to property
+   * names and values.
+   * <p/>
+   * The property names and values are not expected to have any variable replacements done.
+   *
+   * @param identityDescriptors a List of KerberosIdentityDescriptor from which to retrieve configurations
+   * @return a Map of configuration types to property name/value pairs (as a Map)
+   */
+  Map<String, Map<String, String>> getIdentityConfigurations(List<KerberosIdentityDescriptor> identityDescriptors);
+
+  /**
    * Returns the active identities for the named cluster.  Results are filtered by host, service,
    * and/or component; and grouped by host.
    * <p/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
index 6f0fa27..019fb3b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java
@@ -30,6 +30,9 @@ import org.apache.ambari.server.actionmanager.RequestFactory;
 import org.apache.ambari.server.actionmanager.Stage;
 import org.apache.ambari.server.actionmanager.StageFactory;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
 import org.apache.ambari.server.controller.internal.RequestImpl;
@@ -204,6 +207,9 @@ public class KerberosHelperImpl implements KerberosHelper {
   @Inject
   private CredentialStoreService credentialStoreService;
 
+  @Inject
+  private StackAdvisorHelper stackAdvisorHelper;
+
   /**
    * Used to get kerberos descriptors associated with the cluster or stack.
    * Currently not available via injection.
@@ -310,7 +316,7 @@ public class KerberosHelperImpl implements KerberosHelper {
     Map<String, Map<String, String>> existingConfigurations = calculateExistingConfigurations(cluster, hostName);
 
     Map<String, Map<String, String>> updates = getServiceConfigurationUpdates(cluster,
-        existingConfigurations, Collections.singleton(serviceName), serviceAlreadyExists(cluster, serviceComponentHost));
+        existingConfigurations, Collections.singleton(serviceName), serviceAlreadyExists(cluster, serviceComponentHost), true, true);
 
     for (Map.Entry<String, Map<String, String>> entry : updates.entrySet()) {
       configHelper.updateConfigType(cluster, ambariManagementController, entry.getKey(), entry.getValue(), null,
@@ -335,8 +341,12 @@ public class KerberosHelperImpl implements KerberosHelper {
   }
 
   @Override
-  public Map<String, Map<String, String>> getServiceConfigurationUpdates(Cluster cluster, Map<String, Map<String, String>> existingConfigurations, Set<String> services,
-                                                                         boolean serviceAlreadyExists)
+  public Map<String, Map<String, String>> getServiceConfigurationUpdates(Cluster cluster,
+                                                                         Map<String, Map<String, String>> existingConfigurations,
+                                                                         Set<String> services,
+                                                                         boolean serviceAlreadyExists,
+                                                                         boolean kerberosEnabled,
+                                                                         boolean applyStackAdvisorUpdates)
       throws KerberosInvalidConfigurationException, AmbariException {
 
     Map<String, Map<String, String>> kerberosConfigurations = new HashMap<String, Map<String, String>>();
@@ -347,6 +357,8 @@ public class KerberosHelperImpl implements KerberosHelper {
     Map<String, Map<String, String>> configurations = addAdditionalConfigurations(cluster,
         deepCopy(existingConfigurations), null, kerberosDescriptorProperties);
 
+    Map<String, Set<String>> propertiesToIgnore = new HashMap<String, Set<String>>();
+
     for (String serviceName : services) {
       // Set properties...
       KerberosServiceDescriptor serviceDescriptor = kerberosDescriptor.getService(serviceName);
@@ -356,21 +368,42 @@ public class KerberosHelperImpl implements KerberosHelper {
         for (KerberosComponentDescriptor componentDescriptor : componentDescriptors.values()) {
           if (componentDescriptor != null) {
             Map<String, Map<String, String>> identityConfigurations;
-            List<KerberosIdentityDescriptor> identities;
 
-            identities = serviceDescriptor.getIdentities(true);
-            identityConfigurations = getConfigurations(identities);
+            identityConfigurations = getIdentityConfigurations(serviceDescriptor.getIdentities(true));
             if (identityConfigurations != null) {
               for (Map.Entry<String, Map<String, String>> entry : identityConfigurations.entrySet()) {
-                mergeConfigurations(kerberosConfigurations, entry.getKey(), entry.getValue(), configurations);
+                String configType = entry.getKey();
+                Map<String, String> properties = entry.getValue();
+
+                mergeConfigurations(kerberosConfigurations, configType, entry.getValue(), configurations);
+
+                if ((properties != null) && !properties.isEmpty()) {
+                  Set<String> propertyNames = propertiesToIgnore.get(configType);
+                  if (propertyNames == null) {
+                    propertyNames = new HashSet<String>();
+                    propertiesToIgnore.put(configType, propertyNames);
+                  }
+                  propertyNames.addAll(properties.keySet());
+                }
               }
             }
 
-            identities = componentDescriptor.getIdentities(true);
-            identityConfigurations = getConfigurations(identities);
+            identityConfigurations = getIdentityConfigurations(componentDescriptor.getIdentities(true));
             if (identityConfigurations != null) {
               for (Map.Entry<String, Map<String, String>> entry : identityConfigurations.entrySet()) {
-                mergeConfigurations(kerberosConfigurations, entry.getKey(), entry.getValue(), configurations);
+                String configType = entry.getKey();
+                Map<String, String> properties = entry.getValue();
+
+                mergeConfigurations(kerberosConfigurations, configType, entry.getValue(), configurations);
+
+                if ((properties != null) && !properties.isEmpty()) {
+                  Set<String> propertyNames = propertiesToIgnore.get(configType);
+                  if (propertyNames == null) {
+                    propertyNames = new HashSet<String>();
+                    propertiesToIgnore.put(configType, propertyNames);
+                  }
+                  propertyNames.addAll(properties.keySet());
+                }
               }
             }
 
@@ -383,6 +416,151 @@ public class KerberosHelperImpl implements KerberosHelper {
 
     setAuthToLocalRules(kerberosDescriptor, cluster, kerberosDetails.getDefaultRealm(), configurations, kerberosConfigurations);
 
+    return (applyStackAdvisorUpdates)
+        ? applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations, propertiesToIgnore, kerberosEnabled)
+        : kerberosConfigurations;
+  }
+
+  public Map<String, Map<String, String>> applyStackAdvisorUpdates(Cluster cluster, Set<String> services,
+                                                                    Map<String, Map<String, String>> existingConfigurations,
+                                                                    Map<String, Map<String, String>> kerberosConfigurations,
+                                                                    Map<String, Set<String>> propertiesToIgnore,
+                                                                   boolean kerberosEnabled) throws AmbariException {
+
+    StackId stackVersion = cluster.getCurrentStackVersion();
+
+    List<String> hostNames = new ArrayList<String>();
+    Collection<Host> hosts = cluster.getHosts();
+
+    if (hosts != null) {
+      for (Host host : hosts) {
+        hostNames.add(host.getHostName());
+      }
+    }
+
+    // Don't actually call the stack advisor if no hosts are in the cluster, else the stack advisor
+    // will throw a StackAdvisorException stating "Hosts and services must not be empty".
+    // This could happen when enabling Kerberos while installing a cluster via Blueprints due to the
+    // way hosts are discovered during the install process.
+    if(!hostNames.isEmpty()) {
+      Map<String, Map<String, Map<String, String>>> requestConfigurations = new HashMap<String, Map<String, Map<String, String>>>();
+      if (existingConfigurations != null) {
+        for (Map.Entry<String, Map<String, String>> configuration : existingConfigurations.entrySet()) {
+          Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>();
+          String configType = configuration.getKey();
+          Map<String, String> configurationProperties = configuration.getValue();
+
+          if (configurationProperties == null) {
+            configurationProperties = Collections.emptyMap();
+          }
+
+          if ("cluster-env".equals(configType)) {
+            configurationProperties = new HashMap<String, String>(configurationProperties);
+            configurationProperties.put("security_enabled", (kerberosEnabled) ? "true" : "false");
+          }
+
+          properties.put("properties", configurationProperties);
+          requestConfigurations.put(configType, properties);
+        }
+      }
+
+      // Apply the current Kerberos properties...
+      for (Map.Entry<String, Map<String, String>> configuration : kerberosConfigurations.entrySet()) {
+        String configType = configuration.getKey();
+        Map<String, String> configurationProperties = configuration.getValue();
+
+        if ((configurationProperties != null) && !configurationProperties.isEmpty()) {
+          Map<String, Map<String, String>> requestConfiguration = requestConfigurations.get(configType);
+
+          if (requestConfiguration == null) {
+            requestConfiguration = new HashMap<String, Map<String, String>>();
+            requestConfigurations.put(configType, requestConfiguration);
+          }
+
+          Map<String, String> requestConfigurationProperties = requestConfiguration.get("properties");
+          if (requestConfigurationProperties == null) {
+            requestConfigurationProperties = new HashMap<String, String>();
+          } else {
+            requestConfigurationProperties = new HashMap<String, String>(requestConfigurationProperties);
+          }
+
+          requestConfigurationProperties.putAll(configurationProperties);
+          requestConfiguration.put("properties", requestConfigurationProperties);
+        }
+      }
+
+      StackAdvisorRequest request = StackAdvisorRequest.StackAdvisorRequestBuilder
+          .forStack(stackVersion.getStackName(), stackVersion.getStackVersion())
+          .forServices(new ArrayList<String>(services))
+          .forHosts(hostNames)
+          .withConfigurations(requestConfigurations)
+          .ofType(StackAdvisorRequest.StackAdvisorRequestType.CONFIGURATIONS)
+          .build();
+
+      try {
+        RecommendationResponse response = stackAdvisorHelper.recommend(request);
+
+        RecommendationResponse.Recommendation recommendation = (response == null) ? null : response.getRecommendations();
+        RecommendationResponse.Blueprint blueprint = (recommendation == null) ? null : recommendation.getBlueprint();
+        Map<String, RecommendationResponse.BlueprintConfigurations> configurations = (blueprint == null) ? null : blueprint.getConfigurations();
+
+        if (configurations != null) {
+          for (Map.Entry<String, RecommendationResponse.BlueprintConfigurations> configuration : configurations.entrySet()) {
+            String configType = configuration.getKey();
+            Map<String, String> recommendedConfigProperties = configuration.getValue().getProperties();
+            Map<String, String> existingConfigProperties = (existingConfigurations == null) ? null : existingConfigurations.get(configType);
+            Map<String, String> kerberosConfigProperties = kerberosConfigurations.get(configType);
+            Set<String> ignoreProperties = (propertiesToIgnore == null) ? null : propertiesToIgnore.get(configType);
+
+            for (Map.Entry<String, String> property : recommendedConfigProperties.entrySet()) {
+              String propertyName = property.getKey();
+
+              if ((ignoreProperties == null) || !ignoreProperties.contains(propertyName)) {
+                String recommendedValue = property.getValue();
+
+                if (kerberosConfigProperties == null) {
+                  // There is no explicit update for this property from the Kerberos Descriptor...
+                  // add the config and property if it also does not exist in the existing configurations
+                  if ((existingConfigProperties == null) || !existingConfigProperties.containsKey(propertyName)) {
+                    LOG.debug("Adding Kerberos configuration based on StackAdvisor recommendation:" +
+                            "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
+                        configType, propertyName, recommendedValue);
+
+                    HashMap<String, String> properties = new HashMap<String, String>();
+                    properties.put(propertyName, recommendedValue);
+                    kerberosConfigurations.put(configType, properties);
+                  }
+                } else {
+                  String value = kerberosConfigProperties.get(propertyName);
+                  if (value == null) {
+                    // There is no explicit update for this property from the Kerberos Descriptor...
+                    // add the property if it also does not exist in the existing configurations
+                    if ((existingConfigProperties == null) || !existingConfigProperties.containsKey(propertyName)) {
+                      LOG.debug("Adding Kerberos configuration based on StackAdvisor recommendation:" +
+                              "\n\tConfigType: {}\n\tProperty: {}\n\tValue: {}",
+                          configType, propertyName, recommendedValue);
+
+                      kerberosConfigProperties.put(propertyName, recommendedValue);
+                    }
+                  } else if (!value.equals(recommendedValue)) {
+                    // If the recommended value is a change, automatically change it.
+                    LOG.debug("Updating Kerberos configuration based on StackAdvisor recommendation:" +
+                            "\n\tConfigType: {}\n\tProperty: {}\n\tOld Value: {}\n\tNew Value: {}",
+                        configType, propertyName, value, recommendedValue);
+
+                    kerberosConfigProperties.put(propertyName, recommendedValue);
+                  }
+                }
+              }
+            }
+          }
+        }
+
+      } catch (Exception e) {
+        throw new AmbariException(e.getMessage(), e);
+      }
+    }
+
     return kerberosConfigurations;
   }
 
@@ -1911,16 +2089,8 @@ public class KerberosHelperImpl implements KerberosHelper {
         : !"false".equalsIgnoreCase(value);
   }
 
-  /**
-   * Given a list of KerberosIdentityDescriptors, returns a Map fo configuration types to property
-   * names and values.
-   * <p/>
-   * The property names and values are not expected to have any variable replacements done.
-   *
-   * @param identityDescriptors a List of KerberosIdentityDescriptor from which to retrieve configurations
-   * @return a Map of configuration types to property name/value pairs (as a Map)
-   */
-  private Map<String, Map<String, String>> getConfigurations(List<KerberosIdentityDescriptor> identityDescriptors) {
+  @Override
+  public Map<String, Map<String, String>> getIdentityConfigurations(List<KerberosIdentityDescriptor> identityDescriptors) {
     Map<String, Map<String, String>> map = new HashMap<String, Map<String, String>>();
 
     if (identityDescriptors != null) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
index dcedd51..9f42901 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java
@@ -2459,11 +2459,6 @@ public class BlueprintConfigurationProcessor {
       }
 
     }
-
-    if(clusterTopology.isClusterKerberosEnabled()) {
-      configuration.setProperty(CLUSTER_ENV_CONFIG_TYPE_NAME, "security_enabled", "true");
-      configTypesUpdated.add(CLUSTER_ENV_CONFIG_TYPE_NAME);
-    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
index 4c241d1..00b8a74 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java
@@ -36,12 +36,14 @@ import org.apache.ambari.server.actionmanager.Request;
 import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.agent.ExecutionCommand;
 import org.apache.ambari.server.controller.internal.CalculatedStatus;
+import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken;
 import org.apache.ambari.server.utils.StageUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.inject.Inject;
 import com.google.inject.Injector;
+import org.springframework.security.core.context.SecurityContextHolder;
 
 /**
  * Server Action Executor used to execute server-side actions (or tasks)
@@ -471,6 +473,12 @@ public class ServerActionExecutor {
       try {
         LOG.debug("Executing task #{}", taskId);
 
+        // Set an internal administrator user to be the authenticated user in the event an
+        // authorization check is needed while performing a server-side action.
+        InternalAuthenticationToken authentication = new InternalAuthenticationToken("server_action_executor");
+        authentication.setAuthenticated(true);
+        SecurityContextHolder.getContext().setAuthentication(authentication);
+
         commandReport = execute(hostRoleCommand, executionCommand);
 
         LOG.debug("Task #{} completed execution with status of {}",

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
index 359e651..3a945ad 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/AbstractPrepareKerberosServerAction.java
@@ -36,9 +36,9 @@ import org.slf4j.LoggerFactory;
 import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Type;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -62,7 +62,8 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
 
   protected void processServiceComponentHosts(Cluster cluster, KerberosDescriptor kerberosDescriptor, List<ServiceComponentHost> schToProcess,
                                               Collection<String> identityFilter, String dataDirectory,
-                                              Map<String, Map<String, String>> kerberosConfigurations) throws AmbariException {
+                                              Map<String, Map<String, String>> kerberosConfigurations,
+                                              boolean kerberosEnabled) throws AmbariException {
 
     actionLog.writeStdOut("Processing Kerberos identities and configurations");
 
@@ -76,38 +77,35 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
       // Create the file used to store details about principals and keytabs to create
       File identityDataFile = new File(dataDirectory, KerberosIdentityDataFileWriter.DATA_FILE_NAME);
 
-      // Group ServiceComponentHosts with their relevant hosts so we can create the relevant host-based
-      // configurations once per host, rather than for every ServiceComponentHost we encounter
-      Map<String, List<ServiceComponentHost>> hostServiceComponentHosts = new HashMap<String, List<ServiceComponentHost>>();
-      for (ServiceComponentHost sch : schToProcess) {
-        String hostName = sch.getHostName();
-        List<ServiceComponentHost> serviceComponentHosts = hostServiceComponentHosts.get(hostName);
+      Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
+      KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter;
 
-        if (serviceComponentHosts == null) {
-          serviceComponentHosts = new ArrayList<ServiceComponentHost>();
-          hostServiceComponentHosts.put(hostName, serviceComponentHosts);
-        }
+      // Calculate the current host-specific configurations. These will be used to replace
+      // variables within the Kerberos descriptor data
+      Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, null, kerberosDescriptorProperties);
 
-        serviceComponentHosts.add(sch);
+      actionLog.writeStdOut(String.format("Writing Kerberos identity data metadata file to %s", identityDataFile.getAbsolutePath()));
+      try {
+        kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
+      } catch (IOException e) {
+        String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
+        LOG.error(message, e);
+        actionLog.writeStdOut(message);
+        actionLog.writeStdErr(message + "\n" + e.getLocalizedMessage());
+        throw new AmbariException(message, e);
       }
 
-      Map<String, String> kerberosDescriptorProperties = kerberosDescriptor.getProperties();
-      KerberosIdentityDataFileWriter kerberosIdentityDataFileWriter = null;
-
       try {
-        for (Map.Entry<String, List<ServiceComponentHost>> entry : hostServiceComponentHosts.entrySet()) {
-          String hostName = entry.getKey();
-          List<ServiceComponentHost> serviceComponentHosts = entry.getValue();
+        Set<String> services = new HashSet<String>();
+        Map<String, Set<String>> propertiesToIgnore = null;
 
-          // Calculate the current host-specific configurations. These will be used to replace
-          // variables within the Kerberos descriptor data
-          Map<String, Map<String, String>> configurations = kerberosHelper.calculateConfigurations(cluster, hostName, kerberosDescriptorProperties);
+        // Iterate over the components installed on the current host to get the service and
+        // component-level Kerberos descriptors in order to determine which principals,
+        // keytab files, and configurations need to be created or updated.
+        for (ServiceComponentHost sch : schToProcess) {
+          String hostName = sch.getHostName();
 
           try {
-            // Iterate over the components installed on the current host to get the service and
-            // component-level Kerberos descriptors in order to determine which principals,
-            // keytab files, and configurations need to be created or updated.
-            for (ServiceComponentHost sch : serviceComponentHosts) {
               String serviceName = sch.getServiceName();
               String componentName = sch.getServiceComponentName();
 
@@ -116,15 +114,10 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
               if (serviceDescriptor != null) {
                 List<KerberosIdentityDescriptor> serviceIdentities = serviceDescriptor.getIdentities(true);
 
-                // Lazily create the KerberosIdentityDataFileWriter instance...
-                if (kerberosIdentityDataFileWriter == null) {
-                  actionLog.writeStdOut(String.format("Writing Kerberos identity data metadata file to %s", identityDataFile.getAbsolutePath()));
-                  kerberosIdentityDataFileWriter = kerberosIdentityDataFileWriterFactory.createKerberosIdentityDataFileWriter(identityDataFile);
-                }
-
                 // Add service-level principals (and keytabs)
                 kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, serviceIdentities,
                     identityFilter, hostName, serviceName, componentName, kerberosConfigurations, configurations);
+                propertiesToIgnore = gatherPropertiesToIgnore(serviceIdentities, propertiesToIgnore);
 
                 KerberosComponentDescriptor componentDescriptor = serviceDescriptor.getComponent(componentName);
 
@@ -139,9 +132,11 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
                   // Add component-level principals (and keytabs)
                   kerberosHelper.addIdentities(kerberosIdentityDataFileWriter, componentIdentities,
                       identityFilter, hostName, serviceName, componentName, kerberosConfigurations, configurations);
+                  propertiesToIgnore = gatherPropertiesToIgnore(componentIdentities, propertiesToIgnore);
                 }
               }
-            }
+
+              services.add(serviceName);
           } catch (IOException e) {
             String message = String.format("Failed to write index file - %s", identityDataFile.getAbsolutePath());
             LOG.error(message, e);
@@ -150,6 +145,9 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
             throw new AmbariException(message, e);
           }
         }
+
+        kerberosHelper.applyStackAdvisorUpdates(cluster, services, configurations, kerberosConfigurations,
+            propertiesToIgnore, kerberosEnabled);
       }
       finally {
         if (kerberosIdentityDataFileWriter != null) {
@@ -202,4 +200,30 @@ public abstract class AbstractPrepareKerberosServerAction extends KerberosServer
       return null;
     }
   }
+
+  private Map<String, Set<String>> gatherPropertiesToIgnore(List<KerberosIdentityDescriptor> identities,
+                                                            Map<String, Set<String>> propertiesToIgnore) {
+    Map<String,Map<String,String>> identityConfigurations = kerberosHelper.getIdentityConfigurations(identities);
+    if ((identityConfigurations != null) && !identityConfigurations.isEmpty()) {
+      if(propertiesToIgnore == null) {
+        propertiesToIgnore = new HashMap<String, Set<String>>();
+      }
+
+      for (Map.Entry<String, Map<String, String>> entry : identityConfigurations.entrySet()) {
+        String configType = entry.getKey();
+        Map<String, String> properties = entry.getValue();
+
+        if ((properties != null) && !properties.isEmpty()) {
+          Set<String> propertyNames = propertiesToIgnore.get(configType);
+          if (propertyNames == null) {
+            propertyNames = new HashSet<String>();
+            propertiesToIgnore.put(configType, propertyNames);
+          }
+          propertyNames.addAll(properties.keySet());
+        }
+      }
+    }
+
+    return propertiesToIgnore;
+  }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
index 8ab04ff..f0a5440 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareDisableKerberosServerAction.java
@@ -112,7 +112,7 @@ public class PrepareDisableKerberosServerAction extends AbstractPrepareKerberosS
       actionLog.writeStdOut(String.format("Processing %d components", schCount));
     }
 
-    processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations);
+    processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations, false);
 
     actionLog.writeStdOut("Determining configuration changes");
     // Ensure the cluster-env/security_enabled flag is set properly

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
index 84248dc..a7c3861 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareEnableKerberosServerAction.java
@@ -78,7 +78,7 @@ public class PrepareEnableKerberosServerAction extends PrepareKerberosIdentities
       actionLog.writeStdOut(String.format("Processing %d components", schCount));
     }
 
-    processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations);
+    processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations, true);
     processAuthToLocalRules(cluster, kerberosDescriptor, schToProcess, kerberosConfigurations, getDefaultRealm(commandParameters));
 
     // Ensure the cluster-env/security_enabled flag is set properly

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
index fba3eea..8f8f67d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/kerberos/PrepareKerberosIdentitiesServerAction.java
@@ -35,7 +35,6 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 /**
@@ -94,7 +93,7 @@ public class PrepareKerberosIdentitiesServerAction extends AbstractPrepareKerber
       actionLog.writeStdOut(String.format("Processing %d components", schCount));
     }
 
-    processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations);
+    processServiceComponentHosts(cluster, kerberosDescriptor, schToProcess, identityFilter, dataDirectory, kerberosConfigurations, true);
 
     if ("true".equalsIgnoreCase(getCommandParameterValue(commandParameters, UPDATE_CONFIGURATIONS))) {
       processAuthToLocalRules(cluster, kerberosDescriptor, schToProcess, kerberosConfigurations, getDefaultRealm(commandParameters));

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
index 1e89272..0aae007 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java
@@ -141,7 +141,20 @@ public class ClusterConfigurationRequest {
       // apply Kerberos specific configurations
       Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
         .getServiceConfigurationUpdates(cluster, existingConfigurations,
-        new HashSet<String>(blueprint.getServices()), false);
+        new HashSet<String>(blueprint.getServices()), false, true, false);
+
+      // ******************************************************************************************
+      // Since Kerberos is being enabled, make sure the cluster-env/security_enabled property is
+      // set to "true"
+      Map<String, String> clusterEnv = updatedConfigs.get("cluster-env");
+
+      if(clusterEnv == null) {
+        clusterEnv = new HashMap<String,String>();
+        updatedConfigs.put("cluster-env", clusterEnv);
+      }
+
+      clusterEnv.put("security_enabled", "true");
+      // ******************************************************************************************
 
       for (String configType : updatedConfigs.keySet()) {
         Map<String, String> propertyMap = updatedConfigs.get(configType);
@@ -197,7 +210,7 @@ public class ClusterConfigurationRequest {
       // apply Kerberos specific configurations
       Map<String, Map<String, String>> updatedConfigs = AmbariContext.getController().getKerberosHelper()
         .getServiceConfigurationUpdates(cluster, existingConfigurations,
-          new HashSet<String>(blueprint.getServices()), false);
+          new HashSet<String>(blueprint.getServices()), false, true, false);
 
       // retrieve hostgroup for component names extracted from variables like "{clusterHostInfo.(component_name)
       // _host}"

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index ca7d2dd..da2fc47 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -341,7 +341,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         ranger_admin_host = ranger_admin_hosts[0]
         policymgr_external_url = "%s://%s:%s" % (protocol, ranger_admin_host, port)
 
-    putRangerAdminProperty('policymgr_external_url', policymgr_external_url)
+      putRangerAdminProperty('policymgr_external_url', policymgr_external_url)
 
     rangerServiceVersion = [service['StackServices']['service_version'] for service in services["services"] if service['StackServices']['service_name'] == 'RANGER'][0]
     if rangerServiceVersion == '0.4.0':

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
index d5531a6..60a8dde 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorBlueprintProcessorTest.java
@@ -76,6 +76,7 @@ public class StackAdvisorBlueprintProcessorTest {
     expect(clusterTopology.getHostGroupInfo()).andReturn(createHostGroupInfo()).anyTimes();
     expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
     expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
+    expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
     expect(blueprint.getStack()).andReturn(stack).anyTimes();
     expect(stack.getVersion()).andReturn("2.3").anyTimes();
     expect(stack.getName()).andReturn("HDP").anyTimes();
@@ -104,6 +105,7 @@ public class StackAdvisorBlueprintProcessorTest {
     expect(clusterTopology.getHostGroupInfo()).andReturn(createHostGroupInfo()).anyTimes();
     expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
     expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
+    expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
     expect(blueprint.getStack()).andReturn(stack).anyTimes();
     expect(stack.getVersion()).andReturn("2.3").anyTimes();
     expect(stack.getName()).andReturn("HDP").anyTimes();
@@ -131,6 +133,7 @@ public class StackAdvisorBlueprintProcessorTest {
     expect(clusterTopology.getHostGroupInfo()).andReturn(createHostGroupInfo()).anyTimes();
     expect(clusterTopology.getAdvisedConfigurations()).andReturn(advisedConfigurations).anyTimes();
     expect(clusterTopology.getConfiguration()).andReturn(configuration).anyTimes();
+    expect(clusterTopology.isClusterKerberosEnabled()).andReturn(false).anyTimes();
     expect(blueprint.getStack()).andReturn(stack).anyTimes();
     expect(stack.getVersion()).andReturn("2.3").anyTimes();
     expect(stack.getName()).andReturn("HDP").anyTimes();

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
index 8861549..869d187 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/KerberosHelperTest.java
@@ -30,6 +30,9 @@ import org.apache.ambari.server.actionmanager.RequestFactory;
 import org.apache.ambari.server.actionmanager.Stage;
 import org.apache.ambari.server.actionmanager.StageFactory;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorHelper;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.controller.internal.ArtifactResourceProvider;
 import org.apache.ambari.server.controller.internal.RequestStageContainer;
 import org.apache.ambari.server.controller.spi.ClusterController;
@@ -211,6 +214,7 @@ public class KerberosHelperTest extends EasyMockSupport {
         bind(CredentialStoreService.class).to(CredentialStoreServiceImpl.class);
         bind(CreatePrincipalsServerAction.class).toInstance(createMock(CreatePrincipalsServerAction.class));
         bind(CreateKeytabFilesServerAction.class).toInstance(createMock(CreateKeytabFilesServerAction.class));
+        bind(StackAdvisorHelper.class).toInstance(createMock(StackAdvisorHelper.class));
       }
     });
 
@@ -2168,6 +2172,37 @@ public class KerberosHelperTest extends EasyMockSupport {
     expect(metaInfo.getKerberosDescriptor("HDP", "2.2")).andReturn(kerberosDescriptor).atLeastOnce();
     expect(clusterController.ensureResourceProvider(Resource.Type.Artifact)).andReturn(artifactResourceProvider).atLeastOnce();
 
+    RecommendationResponse.BlueprintConfigurations coreSiteRecommendation = createMock(RecommendationResponse.BlueprintConfigurations.class);
+    expect(coreSiteRecommendation.getProperties()).andReturn(Collections.singletonMap("newPropertyRecommendation", "newPropertyRecommendation"));
+
+    RecommendationResponse.BlueprintConfigurations newTypeRecommendation = createMock(RecommendationResponse.BlueprintConfigurations.class);
+    expect(newTypeRecommendation.getProperties()).andReturn(Collections.singletonMap("newTypeRecommendation", "newTypeRecommendation"));
+
+    RecommendationResponse.BlueprintConfigurations type1Recommendation = createMock(RecommendationResponse.BlueprintConfigurations.class);
+    expect(type1Recommendation.getProperties()).andReturn(Collections.singletonMap("replacement1", "not replaced"));
+
+    RecommendationResponse.BlueprintConfigurations service1SiteRecommendation = createMock(RecommendationResponse.BlueprintConfigurations.class);
+    expect(service1SiteRecommendation.getProperties()).andReturn(Collections.singletonMap("component1b.property", "replaced value"));
+
+    Map<String, RecommendationResponse.BlueprintConfigurations> configurations = new HashMap<String, RecommendationResponse.BlueprintConfigurations>();
+    configurations.put("core-site", coreSiteRecommendation);
+    configurations.put("new-type", newTypeRecommendation);
+    configurations.put("type1", type1Recommendation);
+    configurations.put("service1-site", service1SiteRecommendation);
+
+    RecommendationResponse.Blueprint blueprint = createMock(RecommendationResponse.Blueprint.class);
+    expect(blueprint.getConfigurations()).andReturn(configurations).once();
+
+    RecommendationResponse.Recommendation recommendations = createMock(RecommendationResponse.Recommendation.class);
+    expect(recommendations.getBlueprint()).andReturn(blueprint).once();
+
+    RecommendationResponse recommendationResponse = createMock(RecommendationResponse.class);
+    expect(recommendationResponse.getRecommendations()).andReturn(recommendations).once();
+
+    StackAdvisorHelper stackAdvisorHelper = injector.getInstance(StackAdvisorHelper.class);
+    expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(null).once();
+    expect(stackAdvisorHelper.recommend(anyObject(StackAdvisorRequest.class))).andReturn(recommendationResponse).once();
+
     final Service service1 = createMockService("SERVICE1",
         new HashMap<String, ServiceComponent>() {
           {
@@ -2267,10 +2302,10 @@ public class KerberosHelperTest extends EasyMockSupport {
     injector.getInstance(AmbariMetaInfo.class).init();
 
     Map<String, Map<String, String>> updates1 = kerberosHelper.getServiceConfigurationUpdates(
-        cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE2", "SERVICE3")), false);
+        cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE2", "SERVICE3")), false, true, true);
 
     Map<String, Map<String, String>> updates2 = kerberosHelper.getServiceConfigurationUpdates(
-        cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE3")), false);
+        cluster, existingConfigurations, new HashSet<String>(Arrays.asList("SERVICE1", "SERVICE3")), false, true, true);
 
     verifyAll();
 
@@ -2322,6 +2357,13 @@ public class KerberosHelperTest extends EasyMockSupport {
     assertEquals(expectedUpdates, updates1);
 
     expectedUpdates.remove("service2-site");
+    expectedUpdates.get("core-site").put("newPropertyRecommendation", "newPropertyRecommendation");
+    expectedUpdates.get("service1-site").put("component1b.property", "replaced value");
+    expectedUpdates.put("new-type", new HashMap<String, String>() {
+      {
+        put("newTypeRecommendation", "newTypeRecommendation");
+      }
+    });
 
     assertEquals(expectedUpdates, updates2);
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/f3ffe0c2/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
index d1886b9..7309d5b 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ClusterConfigurationRequestTest.java
@@ -48,6 +48,7 @@ import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.anyString;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.verify;
 
 /**
@@ -145,7 +146,7 @@ public class ClusterConfigurationRequestTest {
     expect(kerberosHelper.ensureHeadlessIdentities(anyObject(Cluster.class), anyObject(Map.class), anyObject
       (Set.class))).andReturn(true).once();
     expect(kerberosHelper.getServiceConfigurationUpdates(anyObject(Cluster.class), anyObject(Map.class), anyObject
-      (Set.class), anyBoolean())).andReturn(kerberosConfig).once();
+      (Set.class), anyBoolean(), anyBoolean(), eq(false))).andReturn(kerberosConfig).once();
 
 
     PowerMock.replay(stack, blueprint, topology, controller, clusters, kerberosHelper, ambariContext,


[11/19] ambari git commit: AMBARI-14676. Create DB model for admin setting (Ajit Kumar via smohanty)

Posted by nc...@apache.org.
AMBARI-14676. Create DB model for admin setting (Ajit Kumar via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/58903d89
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/58903d89
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/58903d89

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 58903d89cf0cd50507b6cfddc25c405d6f725c5c
Parents: cbed640
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Jan 18 20:13:37 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Jan 18 20:13:37 2016 -0800

----------------------------------------------------------------------
 .../src/main/resources/Ambari-DDL-Derby-CREATE.sql     | 12 ++++++++++++
 .../src/main/resources/Ambari-DDL-MySQL-CREATE.sql     | 11 +++++++++++
 .../src/main/resources/Ambari-DDL-Oracle-CREATE.sql    | 11 +++++++++++
 .../src/main/resources/Ambari-DDL-Postgres-CREATE.sql  | 12 ++++++++++++
 .../resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql  | 13 +++++++++++++
 .../main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql   | 10 ++++++++++
 .../src/main/resources/Ambari-DDL-SQLServer-CREATE.sql | 11 +++++++++++
 7 files changed, 80 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
index 72413ce..9fc0a16 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
@@ -655,6 +655,16 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+CREATE TABLE adminsetting (
+  id BIGINT NOT NULL,
+  name VARCHAR(255) NOT NULL UNIQUE,
+  setting_type VARCHAR(255) NOT NULL,
+  content VARCHAR(3000) NOT NULL,
+  updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+  update_timestamp BIGINT NOT NULL,
+  PRIMARY KEY (id)
+);
+
 -- tasks indices --
 CREATE INDEX idx_stage_request_id ON stage (request_id);
 CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -1023,6 +1033,8 @@ INSERT INTO ambari_sequences (sequence_name, sequence_value)
   union all
   select 'topology_host_group_id_seq', 0 FROM SYSIBM.SYSDUMMY1
   union all
+  select 'admin_setting_id_seq', 0 FROM SYSIBM.SYSDUMMY1
+  union all
   select 'hostcomponentstate_id_seq', 0  FROM SYSIBM.SYSDUMMY1;
 
 INSERT INTO adminresourcetype (resource_type_id, resource_type_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
index 280b898..775eef2 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
@@ -662,6 +662,16 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+CREATE TABLE adminsetting (
+  id BIGINT NOT NULL,
+  name VARCHAR(255) NOT NULL UNIQUE,
+  setting_type VARCHAR(255) NOT NULL,
+  content TEXT NOT NULL,
+  updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+  update_timestamp BIGINT NOT NULL,
+  PRIMARY KEY (id)
+);
+
 -- tasks indices --
 CREATE INDEX idx_stage_request_id ON stage (request_id);
 CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -987,6 +997,7 @@ INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_lo
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_logical_task_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_request_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_host_group_id_seq', 0);
+INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('admin_setting_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('hostcomponentstate_id_seq', 0);
 
 insert into adminresourcetype (resource_type_id, resource_type_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
index b2ba909..ea9e85f 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
@@ -651,6 +651,16 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+CREATE TABLE adminsetting (
+  id NUMBER(19) NOT NULL,
+  name VARCHAR(255) NOT NULL UNIQUE,
+  setting_type VARCHAR(255) NOT NULL,
+  content CLOB NOT NULL,
+  updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+  update_timestamp NUMBER(19) NOT NULL,
+  PRIMARY KEY (id)
+);
+
 -- tasks indices --
 CREATE INDEX idx_stage_request_id ON stage (request_id);
 CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -977,6 +987,7 @@ INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_lo
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_logical_task_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_request_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_host_group_id_seq', 0);
+INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('admin_setting_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('hostcomponentstate_id_seq', 0);
 
 INSERT INTO metainfo("metainfo_key", "metainfo_value") values ('version', '${ambariVersion}');

http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
index cec88d3..bb49583 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
@@ -655,6 +655,16 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+CREATE TABLE adminsetting (
+  id BIGINT NOT NULL,
+  name VARCHAR(255) NOT NULL UNIQUE,
+  setting_type VARCHAR(255) NOT NULL,
+  content TEXT NOT NULL,
+  updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+  update_timestamp BIGINT NOT NULL,
+  PRIMARY KEY (id)
+);
+
 -- tasks indices --
 CREATE INDEX idx_stage_request_id ON stage (request_id);
 CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -1023,6 +1033,8 @@ INSERT INTO ambari_sequences (sequence_name, sequence_value)
   union all
   select 'topology_host_group_id_seq', 0
   union all
+  select 'admin_setting_id_seq', 0
+  union all
   select 'hostcomponentstate_id_seq', 0;
 
 INSERT INTO adminresourcetype (resource_type_id, resource_type_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
index 99248bc..149c9cf 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
@@ -736,6 +736,17 @@ CREATE TABLE ambari.topology_logical_task (
 );
 GRANT ALL PRIVILEGES ON TABLE ambari.topology_logical_task TO :username;
 
+CREATE TABLE ambari.adminsetting (
+  id BIGINT NOT NULL,
+  name VARCHAR(255) NOT NULL UNIQUE,
+  setting_type VARCHAR(255) NOT NULL,
+  content TEXT NOT NULL,
+  updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+  update_timestamp BIGINT NOT NULL,
+  PRIMARY KEY (id)
+);
+GRANT ALL PRIVILEGES ON TABLE ambari.adminsetting TO :username;
+
 -- tasks indices --
 CREATE INDEX idx_stage_request_id ON ambari.stage (request_id);
 CREATE INDEX idx_hrc_request_id ON ambari.host_role_command (request_id);
@@ -1121,6 +1132,8 @@ INSERT INTO ambari.ambari_sequences (sequence_name, sequence_value)
   union all
   select 'topology_host_group_id_seq', 0
   union all
+  select 'admin_setting_id_seq', 0
+  union all
   select 'hostcomponentstate_id_seq', 0;
 
 INSERT INTO ambari.adminresourcetype (resource_type_id, resource_type_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
index 7831836..7d88568 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
@@ -652,6 +652,15 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY (id)
 );
 
+CREATE TABLE adminsetting (
+  id NUMERIC(19) NOT NULL,
+  name VARCHAR(255) NOT NULL UNIQUE,
+  setting_type VARCHAR(255) NOT NULL,
+  content TEXT NOT NULL,
+  updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+  update_timestamp NUMERIC(19) NOT NULL,
+  PRIMARY KEY (id)
+);
 -- tasks indices --
 CREATE INDEX idx_stage_request_id ON stage (request_id);
 CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -975,6 +984,7 @@ INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_lo
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_logical_task_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_request_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('topology_host_group_id_seq', 0);
+INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('admin_setting_id_seq', 0);
 INSERT INTO ambari_sequences(sequence_name, sequence_value) values ('hostcomponentstate_id_seq', 0);
 
 insert into adminresourcetype (resource_type_id, resource_type_name)

http://git-wip-us.apache.org/repos/asf/ambari/blob/58903d89/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
index 7f9ea12..4374fc6 100644
--- a/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
+++ b/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
@@ -760,6 +760,16 @@ CREATE TABLE topology_logical_task (
   PRIMARY KEY CLUSTERED (id)
 );
 
+CREATE TABLE adminsetting (
+  id BIGINT NOT NULL,
+  name VARCHAR(255) NOT NULL UNIQUE,
+  setting_type VARCHAR(255) NOT NULL,
+  content TEXT NOT NULL,
+  updated_by VARCHAR(255) NOT NULL DEFAULT '_db',
+  update_timestamp BIGINT NOT NULL,
+  PRIMARY KEY (id)
+);
+
 -- tasks indices --
 CREATE INDEX idx_stage_request_id ON stage (request_id);
 CREATE INDEX idx_hrc_request_id ON host_role_command (request_id);
@@ -1090,6 +1100,7 @@ BEGIN TRANSACTION
     ('topology_logical_task_id_seq', 0),
     ('topology_request_id_seq', 0),
     ('topology_host_group_id_seq', 0),
+    ('admin_setting_id_seq', 0),
     ('hostcomponentstate_id_seq', 0);
 
   insert into adminresourcetype (resource_type_id, resource_type_name)


[03/19] ambari git commit: AMBARI-14704 : Restart storm fails with a metrics storm sink jar related error sometimes - Commit 2. (avijayan)

Posted by nc...@apache.org.
AMBARI-14704 : Restart storm fails with a metrics storm sink jar related error sometimes - Commit 2. (avijayan)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8684e450
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8684e450
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8684e450

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8684e45065b3849b9ba86588309f75c535cde39b
Parents: ba3db6d
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Mon Jan 18 09:22:11 2016 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Mon Jan 18 09:22:15 2016 -0800

----------------------------------------------------------------------
 .../STORM/0.9.1.2.1/package/scripts/storm.py             | 11 -----------
 1 file changed, 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8684e450/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
index 98abb41..43736fa 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
@@ -87,17 +87,6 @@ def storm(name=None):
        group=params.user_group
   )
 
-    # Remove symlinks. They can be there, if you doing upgrade from HDP < 2.2 to HDP >= 2.2
-    Link(format("{storm_lib_dir}/ambari-metrics-storm-sink.jar"),
-         action="delete")
-    # On old HDP 2.1 versions, this symlink may also exist and break EU to newer versions
-    Link("/usr/lib/storm/lib/ambari-metrics-storm-sink.jar", action="delete")
-
-    Execute(format("{sudo} ln -s {metric_collector_sink_jar} {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
-            not_if=format("ls {storm_lib_dir}/ambari-metrics-storm-sink.jar"),
-            only_if=format("ls {metric_collector_sink_jar}")
-    )
-
   File(format("{conf_dir}/storm-env.sh"),
        owner=params.storm_user,
        content=InlineTemplate(params.storm_env_sh_template)


[12/19] ambari git commit: AMBARI-14710. Add retry logic for REST calls between Ambari and Ranger(gautam)

Posted by nc...@apache.org.
AMBARI-14710. Add retry logic for REST calls between Ambari and Ranger(gautam)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/869e0858
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/869e0858
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/869e0858

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 869e0858ccd835baa53ec3b88f50439cdecfd2c7
Parents: 58903d8
Author: Gautam Borad <ga...@apache.org>
Authored: Mon Jan 18 22:24:43 2016 +0530
Committer: Gautam Borad <ga...@apache.org>
Committed: Tue Jan 19 09:56:21 2016 +0530

----------------------------------------------------------------------
 .../libraries/functions/ranger_functions.py                  | 8 +++-----
 .../libraries/functions/ranger_functions_v2.py               | 8 +++-----
 2 files changed, 6 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/869e0858/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
index 543a31f..c59dc8d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions.py
@@ -219,13 +219,11 @@ class Rangeradmin:
       return response_code
     except urllib2.URLError, e:
       if isinstance(e, urllib2.HTTPError):
-        Logger.error("Connection to Ranger Admin failed. Http status code - {0}. \n {1}".format(e.code, e.read()))
+        raise Fail("Connection to Ranger Admin failed. Http status code - {0}. \n {1}".format(e.code, e.read()))
       else:
-        Logger.error("Connection to Ranger Admin failed. Reason - {0}.".format(e.reason))
-      return None
+        raise Fail("Connection to Ranger Admin failed. Reason - {0}.".format(e.reason))
     except httplib.BadStatusLine, e:
-      Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
-      return None
+      raise Fail("Ranger Admin service is not reachable, please restart the service and then try again")
     except TimeoutError:
       raise Fail("Connection to Ranger Admin failed. Reason - timeout")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/869e0858/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
index 80965a6..a72be33 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/ranger_functions_v2.py
@@ -183,13 +183,11 @@ class RangeradminV2:
       return response_code
     except urllib2.URLError, e:
       if isinstance(e, urllib2.HTTPError):
-        Logger.error("Connection failed to Ranger Admin. Http status code - {0}. \n {1}".format(e.code, e.read()))
+        raise Fail("Connection failed to Ranger Admin. Http status code - {0}. \n {1}".format(e.code, e.read()))
       else:
-        Logger.error("Connection failed to Ranger Admin. Reason - {0}.".format(e.reason))
-      return None
+        raise Fail("Connection failed to Ranger Admin. Reason - {0}.".format(e.reason))
     except httplib.BadStatusLine, e:
-      Logger.error("Ranger Admin service is not reachable, please restart the service and then try again")
-      return None
+      raise Fail("Ranger Admin service is not reachable, please restart the service and then try again")
     except TimeoutError:
       raise Fail("Connection failed to Ranger Admin. Reason - timeout")
 


[18/19] ambari git commit: AMBARI-14725. HDP-2.3 to HDP-2.4 Upgrade Pack is missing Mahout and Phoenix Query Server (ncole)

Posted by nc...@apache.org.
AMBARI-14725. HDP-2.3 to HDP-2.4 Upgrade Pack is missing Mahout and Phoenix Query Server (ncole)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/09d3e8bd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/09d3e8bd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/09d3e8bd

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 09d3e8bd636b8a4cf150d3b6b62c9738ee46bc2b
Parents: 07b8a3a
Author: Nate Cole <nc...@hortonworks.com>
Authored: Tue Jan 19 12:48:38 2016 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Tue Jan 19 12:48:38 2016 -0500

----------------------------------------------------------------------
 .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml      | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/09d3e8bd/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index c15769b..e579ee0 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -164,6 +164,7 @@
 
       <service name="HBASE">
         <component>HBASE_REGIONSERVER</component>
+        <component>PHOENIX_QUERY_SERVER</component>
       </service>
 
       <service name="YARN">
@@ -284,6 +285,10 @@
         <component>SQOOP</component>
       </service>
 
+      <service name="MAHOUT">
+        <component>MAHOUT</component>
+      </service>
+
       <service name="HIVE">
         <component>HIVE_CLIENT</component>
         <component>HCAT</component>
@@ -530,6 +535,12 @@
           <task xsi:type="restart-task" />
         </upgrade>
       </component>
+      
+      <component name="PHOENIX_QUERY_SERVER">
+        <upgrade>
+          <task xsi:type="restart-task"/>
+        </upgrade>
+      </component>
     </service>
 
     <service name="TEZ">
@@ -551,6 +562,14 @@
       </component>
     </service>
 
+    <service name="MAHOUT">
+      <component name="MAHOUT">
+        <upgrade>
+          <task xsi:type="restart-task" />
+        </upgrade>
+      </component>
+    </service>
+
     <service name="SQOOP">
       <component name="SQOOP">
         <upgrade>


[17/19] ambari git commit: AMBARI-14719 After enabling NN HA AMS shows critical alert. (ababiichuk)

Posted by nc...@apache.org.
AMBARI-14719 After enabling NN HA AMS shows critical alert. (ababiichuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/07b8a3ab
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/07b8a3ab
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/07b8a3ab

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 07b8a3ab2c8fa72a9f8d63fa5564a1f13835e26c
Parents: 8f6b45d
Author: ababiichuk <ab...@hortonworks.com>
Authored: Tue Jan 19 15:54:51 2016 +0200
Committer: ababiichuk <ab...@hortonworks.com>
Committed: Tue Jan 19 17:20:25 2016 +0200

----------------------------------------------------------------------
 .../configs/config_recommendation_parser.js     |  9 +++++--
 ...onfig_with_override_recommendation_parser.js |  2 +-
 .../mixins/common/configs/enhanced_configs.js   | 13 ++--------
 ambari-web/app/views/common/controls_view.js    |  7 ++---
 .../config_recommendation_parser_test.js        | 27 +++++++++++++++++++-
 5 files changed, 38 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/app/mixins/common/configs/config_recommendation_parser.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/config_recommendation_parser.js b/ambari-web/app/mixins/common/configs/config_recommendation_parser.js
index b5e2f2d..826ce19 100644
--- a/ambari-web/app/mixins/common/configs/config_recommendation_parser.js
+++ b/ambari-web/app/mixins/common/configs/config_recommendation_parser.js
@@ -290,9 +290,14 @@ App.ConfigRecommendationParser = Em.Mixin.create(App.ConfigRecommendations, {
 	 * @param parentProperties
 	 * @param name
 	 * @param fileName
+	 * @param [configGroup]
 	 * @returns {boolean}
 	 */
-	allowUpdateProperty: function (parentProperties, name, fileName) {
-		return true;
+	allowUpdateProperty: function (parentProperties, name, fileName, configGroup) {
+		try {
+			return Em.get(this.getRecommendation(name, fileName, configGroup), 'saveRecommended');
+		} catch (e) {
+			return true;
+		}
 	}
 });
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js b/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js
index 7002110..ad06cde 100644
--- a/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js
+++ b/ambari-web/app/mixins/common/configs/config_with_override_recommendation_parser.js
@@ -46,7 +46,7 @@ App.ConfigWithOverrideRecommendationParser = Em.Mixin.create(App.ConfigRecommend
 	 * @protected
 	 */
 	_updateOverride: function(config, recommendedValue, parentProperties, configGroup) {
-		var updateValue = this.allowUpdateProperty(parentProperties, Em.get(config, 'name'), Em.get(config, 'filename'));
+		var updateValue = this.allowUpdateProperty(parentProperties, Em.get(config, 'name'), Em.get(config, 'filename'), configGroup.get('name'));
 		var override = config.getOverride(configGroup.get('name'));
 		if (override) {
 			this._updateConfigByRecommendation(override, recommendedValue, parentProperties);

http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/app/mixins/common/configs/enhanced_configs.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/mixins/common/configs/enhanced_configs.js b/ambari-web/app/mixins/common/configs/enhanced_configs.js
index 85d157e..303b0c7 100644
--- a/ambari-web/app/mixins/common/configs/enhanced_configs.js
+++ b/ambari-web/app/mixins/common/configs/enhanced_configs.js
@@ -146,7 +146,7 @@ App.EnhancedConfigsMixin = Em.Mixin.create(App.ConfigWithOverrideRecommendationP
   /**
    * sends request to get values for dependent configs
    * @param {{type: string, name: string}[]} changedConfigs - list of changed configs to track recommendations
-   * @param {Function} onComplete
+   * @param {Function} [onComplete]
    * @returns {$.ajax|null}
    */
   loadConfigRecommendations: function(changedConfigs, onComplete) {
@@ -379,15 +379,6 @@ App.EnhancedConfigsMixin = Em.Mixin.create(App.ConfigWithOverrideRecommendationP
    */
   removeCurrentFromDependentList: function (config, saveRecommended) {
     var recommendation = this.getRecommendation(config.get('name'), config.get('filename'), config.get('group.name'));
-    if (recommendation) {
-      try {
-        if (this.saveRecommendation(recommendation)) {
-          this.undoRedoRecommended([recommendation], saveRecommended);
-          this.set('recommendationTimeStamp', (new Date).getTime());
-        }
-      } catch(e) {
-        console.warn(e.message);
-      }
-    }
+    if (recommendation) this.saveRecommendation(recommendation, saveRecommended);
   }
 });

http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/app/views/common/controls_view.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/common/controls_view.js b/ambari-web/app/views/common/controls_view.js
index 48493f0..c9281a8 100644
--- a/ambari-web/app/views/common/controls_view.js
+++ b/ambari-web/app/views/common/controls_view.js
@@ -103,6 +103,7 @@ App.SupportsDependentConfigs = Ember.Mixin.create({
       var saveRecommended = (config.get('value') === config.get('recommendedValue'));
       var type = App.config.getConfigTagFromFileName(config.get('filename'));
       var p = App.configsCollection.getConfig(App.config.configId(name, type));
+      controller.removeCurrentFromDependentList(config, saveRecommended);
        if ((p && Em.get(p, 'propertyDependedBy.length') > 0 || Em.get(p, 'displayType') === 'user') && config.get('oldValue') !== config.get('value')) {
          var old = config.get('oldValue');
          config.set('oldValue', config.get('value'));
@@ -110,11 +111,7 @@ App.SupportsDependentConfigs = Ember.Mixin.create({
            "type": type,
            "name": name,
            "old_value": Em.isNone(old) ? config.get('initialValue') : old
-         }], function() {
-           controller.removeCurrentFromDependentList(config, saveRecommended);
-         });
-      } else {
-        controller.removeCurrentFromDependentList(config, saveRecommended);
+         }]);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/07b8a3ab/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js b/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js
index e895859..35f84cd 100644
--- a/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js
+++ b/ambari-web/test/mixins/common/configs/config_recommendation_parser_test.js
@@ -71,6 +71,7 @@ describe('App.ConfigRecommendationParser', function() {
         sinon.stub(App.configsCollection, 'getConfigByName', function(name, fileName) {
           return { name: name, filename: fileName };
         });
+        sinon.stub(instanceObject, 'allowUpdateProperty').returns(true);
 
         sinon.spy(instanceObject, 'updateCallback');
         sinon.spy(instanceObject, 'removeCallback');
@@ -83,6 +84,7 @@ describe('App.ConfigRecommendationParser', function() {
       afterEach(function() {
         App.configsCollection.getConfigByName.restore();
 
+        instanceObject.allowUpdateProperty.restore();
         instanceObject.updateCallback.restore();
         instanceObject.removeCallback.restore();
         instanceObject.updateBoundariesCallback.restore();
@@ -449,9 +451,32 @@ describe('App.ConfigRecommendationParser', function() {
   });
 
   describe('#allowUpdateProperty', function() {
+
+
+
+    var cases = [{
+      saveRecommended: true
+    },{
+      saveRecommended: false
+    }];
+
+    cases.forEach(function(c) {
+      describe('allowUpdateProperty based on saveRecommended:' + c.saveRecommended, function() {
+        beforeEach(function() {
+          sinon.stub(instanceObject, 'getRecommendation').returns({saveRecommended: c.saveRecommended});
+        });
+        afterEach(function() {
+          instanceObject.getRecommendation.restore();
+        });
+        it('default value for allowUpdateProperty is true', function() {
+          expect(instanceObject.allowUpdateProperty()).to.equal(c.saveRecommended);
+        });
+      });
+    });
+
     it('default value for allowUpdateProperty is true', function() {
       expect(instanceObject.allowUpdateProperty()).to.be.true;
-    })
+    });
   });
 });
 


[02/19] ambari git commit: AMBARI-14705. nodes in non default config group do no get prompted for restart to update stale configs (smohanty)

Posted by nc...@apache.org.
AMBARI-14705. nodes in non default config group do no get prompted for restart to update stale configs (smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba3db6d7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba3db6d7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba3db6d7

Branch: refs/heads/branch-dev-patch-upgrade
Commit: ba3db6d79b7204b6fcf3e105d7aa4a472ca451b0
Parents: 40aa612
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Jan 18 08:32:14 2016 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Jan 18 08:39:13 2016 -0800

----------------------------------------------------------------------
 .../ambari/server/state/ConfigHelper.java       |  7 ++-
 .../server/state/cluster/ClusterImpl.java       | 11 ++++-
 .../ambari/server/state/ConfigHelperTest.java   | 52 +++++++++++++++++---
 3 files changed, 62 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3db6d7/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 83d3a3a..d311017 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -379,6 +379,10 @@ public class ConfigHelper {
     if (stale == null) {
       stale = calculateIsStaleConfigs(sch);
       staleConfigsCache.put(sch, stale);
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("Config staleness for " +
+                  sch.getServiceComponentName() + " on host " + sch.getHostName() + " - " + stale);
+      }
     }
     return stale;
   }
@@ -1134,10 +1138,11 @@ public class ConfigHelper {
    * @return true if the tags are different in any way, even if not-specified
    */
   private boolean isTagChanged(Map<String, String> desiredTags, Map<String, String> actualTags, boolean groupSpecificConfigs) {
-    if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG)) && !groupSpecificConfigs) {
+    if (!actualTags.get(CLUSTER_DEFAULT_TAG).equals(desiredTags.get(CLUSTER_DEFAULT_TAG))) {
       return true;
     }
 
+    // cluster level configs are already compared for staleness, now they match
     // if the host has group specific configs for type we should ignore the cluster level configs and compare specifics
     if (groupSpecificConfigs) {
       actualTags.remove(CLUSTER_DEFAULT_TAG);

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3db6d7/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
index 14223fe..1d20322 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
@@ -487,9 +487,18 @@ public class ClusterImpl implements Cluster {
     loadConfigGroups();
     clusterGlobalLock.writeLock().lock();
     try {
+      String hostList = "";
+      if(LOG.isDebugEnabled()) {
+        if (configGroup.getHosts() != null) {
+          for (Host host : configGroup.getHosts().values()) {
+            hostList += host.getHostName() + ", ";
+          }
+        }
+      }
+
       LOG.debug("Adding a new Config group" + ", clusterName = "
         + getClusterName() + ", groupName = " + configGroup.getName()
-        + ", tag = " + configGroup.getTag());
+        + ", tag = " + configGroup.getTag() + " with hosts " + hostList);
 
       if (clusterConfigGroups.containsKey(configGroup.getId())) {
         // The loadConfigGroups will load all groups to memory

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3db6d7/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index bf023c2..98424b7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -62,6 +62,8 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.runners.Enclosed;
 import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
@@ -74,6 +76,7 @@ import org.springframework.security.core.context.SecurityContextHolder;
 @RunWith(Enclosed.class)
 public class ConfigHelperTest {
   public static class RunWithInMemoryDefaultTestModule {
+    private final static Logger LOG = LoggerFactory.getLogger(ConfigHelperTest.class);
     private Clusters clusters;
     private Injector injector;
     private String clusterName;
@@ -220,6 +223,8 @@ public class ConfigHelperTest {
 
       ConfigGroup configGroup = configGroupFactory.createNew(cluster, name,
           tag, "", configMap, hostMap);
+      LOG.info("Config group created with tag " + tag);
+      configGroup.setTag(tag);
 
       configGroup.persist();
       cluster.addConfigGroup(configGroup);
@@ -660,17 +665,19 @@ public class ConfigHelperTest {
       // Put a different version to check for change
       hc.setDefaultVersionTag("version2");
       schReturn.put("flume-conf", hc);
+
       // set up mocks
       ServiceComponentHost sch = createNiceMock(ServiceComponentHost.class);
       // set up expectations
-      expect(sch.getActualConfigs()).andReturn(schReturn).times(3);
-      expect(sch.getHostName()).andReturn("h1").times(6);
-      expect(sch.getClusterId()).andReturn(1l).times(3);
-      expect(sch.getServiceName()).andReturn("FLUME").times(3);
-      expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").times(3);
+      expect(sch.getActualConfigs()).andReturn(schReturn).times(6);
+      expect(sch.getHostName()).andReturn("h1").anyTimes();
+      expect(sch.getClusterId()).andReturn(1l).anyTimes();
+      expect(sch.getServiceName()).andReturn("FLUME").anyTimes();
+      expect(sch.getServiceComponentName()).andReturn("FLUME_HANDLER").anyTimes();
       replay(sch);
       // Cluster level config changes
       Assert.assertTrue(configHelper.isStaleConfigs(sch));
+
       HostConfig hc2 = new HostConfig();
       hc2.setDefaultVersionTag("version1");
       schReturn.put("flume-conf", hc2);
@@ -678,13 +685,46 @@ public class ConfigHelperTest {
       configHelper.invalidateStaleConfigsCache();
       // Cluster level same configs
       Assert.assertFalse(configHelper.isStaleConfigs(sch));
+
       // Cluster level same configs but group specific configs for host have been updated
       List<String> hosts = new ArrayList<String>();
       hosts.add("h1");
       List<Config> configs = new ArrayList<Config>();
-      configs.add(new ConfigImpl("flume-conf"));
+      ConfigImpl configImpl = new ConfigImpl("flume-conf");
+      configImpl.setTag("FLUME1");
+      configs.add(configImpl);
       addConfigGroup("configGroup1", "FLUME", hosts, configs);
+
+      // config group added for host - expect staleness
       Assert.assertTrue(configHelper.isStaleConfigs(sch));
+
+      HostConfig hc3 = new HostConfig();
+      hc3.setDefaultVersionTag("version1");
+      hc3.getConfigGroupOverrides().put(1l, "FLUME1");
+      schReturn.put("flume-conf", hc3);
+      configHelper.invalidateStaleConfigsCache();
+
+      // version1 and FLUME1 - stale=false
+      Assert.assertFalse(configHelper.isStaleConfigs(sch));
+
+      HostConfig hc4 = new HostConfig();
+      hc4.setDefaultVersionTag("version1");
+      hc4.getConfigGroupOverrides().put(1l, "FLUME2");
+      schReturn.put("flume-conf", hc4);
+      configHelper.invalidateStaleConfigsCache();
+
+      // version1 and FLUME2 - stale=true
+      Assert.assertTrue(configHelper.isStaleConfigs(sch));
+
+      HostConfig hc5 = new HostConfig();
+      hc5.setDefaultVersionTag("version3");
+      hc5.getConfigGroupOverrides().put(1l, "FLUME1");
+      schReturn.put("flume-conf", hc5);
+      configHelper.invalidateStaleConfigsCache();
+
+      // version3 and FLUME1 - stale=true
+      Assert.assertTrue(configHelper.isStaleConfigs(sch));
+
       verify(sch);
     }
   }


[08/19] ambari git commit: AMBARI-14656: dfs.allow.truncate property check for Add Service wizard of HAWQ

Posted by nc...@apache.org.
AMBARI-14656: dfs.allow.truncate property check for Add Service wizard of HAWQ


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/132c5350
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/132c5350
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/132c5350

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 132c5350e40be4964b88d6f3feca275457523d92
Parents: 21b6b31
Author: Jun Aoki <ja...@apache.org>
Authored: Mon Jan 18 13:10:51 2016 -0800
Committer: Jun Aoki <ja...@apache.org>
Committed: Mon Jan 18 13:10:51 2016 -0800

----------------------------------------------------------------------
 .../stacks/HDP/2.3/services/stack_advisor.py    |  8 +++
 .../stacks/2.3/common/test_stack_advisor.py     | 60 ++++++++++++++++++++
 2 files changed, 68 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/132c5350/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
index ed241ff..0544f5a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/stack_advisor.py
@@ -703,6 +703,14 @@ class HDP23StackAdvisor(HDP22StackAdvisor):
         validationItems.append({"config-name": 'dfs.namenode.inode.attributes.provider.class',
                                     "item": self.getWarnItem(
                                       "dfs.namenode.inode.attributes.provider.class needs to be set to 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer' if Ranger HDFS Plugin is enabled.")})
+
+    # Check if dfs.allow.truncate is true
+    if "HAWQ" in servicesList and \
+        not ("dfs.allow.truncate" in services["configurations"]["hdfs-site"]["properties"] and \
+        services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"].lower() == 'true'):
+        validationItems.append({"config-name": "dfs.allow.truncate",
+                                "item": self.getWarnItem("HAWQ requires dfs.allow.truncate in hdfs-site.xml set to True.")})
+
     validationProblems = self.toConfigurationValidationProblems(validationItems, "hdfs-site")
     validationProblems.extend(parentValidationProblems)
     return validationProblems

http://git-wip-us.apache.org/repos/asf/ambari/blob/132c5350/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
index 7e59a6f..362a7cd 100644
--- a/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py
@@ -1521,3 +1521,63 @@ class TestHDP23StackAdvisor(TestCase):
     recommendedConfigurations = {}
     self.stackAdvisor.recommendRangerConfigurations(recommendedConfigurations, clusterData, services, None)
     self.assertEquals(recommendedConfigurations['ranger-admin-site']['properties']['ranger.audit.solr.zookeepers'], 'NONE')
+
+  def test_validateHDFSConfigurations(self):
+    properties = {}
+    recommendedDefaults = {}
+    configurations = {
+      "core-site": {"properties": {}},
+    }
+    services = self.load_json("services-hawq-3-hosts.json")
+    services["configurations"]["hdfs-site"] = {}
+    services["configurations"]["hdfs-site"]["properties"] = {}
+    hosts = {}
+
+    expected_warning = {
+        'config-type':'hdfs-site',
+        'message':'HAWQ requires dfs.allow.truncate in hdfs-site.xml set to True.',
+        'type':'configuration',
+        'config-name':'dfs.allow.truncate',
+        'level':'WARN'
+    }
+
+    # Test following cases:
+    # when HAWQ is being installed and dfs.allow.truncate is not set at all, warning
+    # when HAWQ is being installed and dfs.allow.truncate is set to True, no warning
+    # when HAWQ is being installed and dfs.allow.truncate is not set to True, warning
+    # when HAWQ is not installed and dfs.allow.truncate is not set at all, no warning
+    # when HAWQ is not installed and dfs.allow.truncate is set to True, no warning
+    # when HAWQ is not installed and dfs.allow.truncate is not set to True, no warning
+    # 1
+    problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 1)
+    self.assertEqual(problems[0], expected_warning)
+
+    # 2
+    services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "True"
+    problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)
+
+    # 3
+    services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "false"
+    problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 1)
+    self.assertEqual(problems[0], expected_warning)
+
+    del services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"]
+    servicesElementWithoutHAWQ = [service for service in services["services"] if service["StackServices"]["service_name"] != "HAWQ"]
+    services["services"] = servicesElementWithoutHAWQ
+
+    # 4
+    problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)
+
+    # 5
+    services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "True"
+    problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)
+
+    # 6
+    services["configurations"]["hdfs-site"]["properties"]["dfs.allow.truncate"] = "false"
+    problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, services, hosts)
+    self.assertEqual(len(problems), 0)


[19/19] ambari git commit: Merge branch 'trunk' into branch-dev-patch-upgrade

Posted by nc...@apache.org.
Merge branch 'trunk' into branch-dev-patch-upgrade


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4b36397f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4b36397f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4b36397f

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 4b36397ffd14f2afed81f0c8d16ce2d2e709530f
Parents: f40df6b 09d3e8b
Author: Nate Cole <nc...@hortonworks.com>
Authored: Tue Jan 19 13:26:47 2016 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Tue Jan 19 13:26:47 2016 -0500

----------------------------------------------------------------------
 .../python/resource_management/core/shell.py    |  10 +-
 .../libraries/functions/ranger_functions.py     |   8 +-
 .../libraries/functions/ranger_functions_v2.py  |   8 +-
 .../server/controller/KerberosHelper.java       |  41 +++-
 .../server/controller/KerberosHelperImpl.java   | 210 +++++++++++++++++--
 .../BlueprintConfigurationProcessor.java        |   5 -
 .../serveraction/ServerActionExecutor.java      |   8 +
 .../AbstractPrepareKerberosServerAction.java    |  90 +++++---
 .../PrepareDisableKerberosServerAction.java     |   2 +-
 .../PrepareEnableKerberosServerAction.java      |   2 +-
 .../PrepareKerberosIdentitiesServerAction.java  |   3 +-
 .../upgrades/SparkShufflePropertyConfig.java    |  14 +-
 .../ambari/server/state/ConfigHelper.java       |   7 +-
 .../server/state/cluster/ClusterImpl.java       |  11 +-
 .../server/state/stack/upgrade/Grouping.java    | 125 +++++++----
 .../stack/upgrade/StageWrapperBuilder.java      |   2 +-
 .../ambari/server/state/stack/upgrade/Task.java |   8 +
 .../server/state/stack/upgrade/TaskWrapper.java |  13 ++
 .../state/stack/upgrade/TaskWrapperBuilder.java |  10 +
 .../svccomphost/ServiceComponentHostImpl.java   |  65 ++++--
 .../topology/ClusterConfigurationRequest.java   |  17 +-
 .../python/ambari_server/serverConfiguration.py |  15 +-
 .../main/resources/Ambari-DDL-Derby-CREATE.sql  |  12 ++
 .../main/resources/Ambari-DDL-MySQL-CREATE.sql  |  11 +
 .../main/resources/Ambari-DDL-Oracle-CREATE.sql |  11 +
 .../resources/Ambari-DDL-Postgres-CREATE.sql    |  12 ++
 .../Ambari-DDL-Postgres-EMBEDDED-CREATE.sql     |  13 ++
 .../resources/Ambari-DDL-SQLAnywhere-CREATE.sql |  10 +
 .../resources/Ambari-DDL-SQLServer-CREATE.sql   |  11 +
 .../0.96.0.2.0/configuration/hbase-env.xml      |   2 +-
 .../HBASE/0.96.0.2.0/package/scripts/hbase.py   |   5 +-
 .../0.96.0.2.0/package/scripts/params_linux.py  |   2 -
 .../STORM/0.9.1.2.1/package/scripts/storm.py    |  11 -
 .../stacks/HDP/2.0.6/services/stack_advisor.py  |   2 +-
 .../HDP/2.1/upgrades/nonrolling-upgrade-2.3.xml |   5 +-
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.2.xml |  29 +--
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.3.xml |  29 +--
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml |  31 +--
 .../stacks/HDP/2.2/upgrades/upgrade-2.2.xml     |   8 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml     |   8 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml     |  10 +-
 .../stacks/HDP/2.3/role_command_order.json      |   4 +-
 .../stacks/HDP/2.3/services/stack_advisor.py    |   8 +
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.3.xml |  29 +--
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml |  31 +--
 .../stacks/HDP/2.3/upgrades/upgrade-2.3.xml     |   8 +-
 .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml     |  29 ++-
 .../services/YARN/configuration/yarn-site.xml   |  14 +-
 .../stacks/HDP/2.4/services/stack_advisor.py    |  78 +------
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.4.xml |  29 +--
 .../stacks/HDP/2.4/upgrades/upgrade-2.4.xml     |   8 +-
 .../StackAdvisorBlueprintProcessorTest.java     |   3 +
 .../server/controller/KerberosHelperTest.java   |  46 +++-
 .../SparkShufflePropertyConfigTest.java         |  43 +---
 .../ambari/server/state/ConfigHelperTest.java   |  52 ++++-
 .../ambari/server/state/UpgradeHelperTest.java  | 168 ++++++++++++---
 .../ClusterConfigurationRequestTest.java        |   3 +-
 .../src/test/python/TestAmbariServer.py         |  38 +++-
 .../stacks/2.0.6/HBASE/test_hbase_client.py     |   4 -
 .../stacks/2.0.6/HBASE/test_hbase_master.py     |   6 -
 .../2.0.6/HBASE/test_hbase_regionserver.py      |   8 -
 .../2.0.6/HBASE/test_phoenix_queryserver.py     |   4 -
 .../stacks/2.3/common/test_stack_advisor.py     |  60 ++++++
 .../HDP/2.0.7/services/OOZIE/metainfo.xml       | 176 ++++++++++++++++
 .../HDP/2.1.1/upgrades/upgrade_bucket_test.xml  |  20 +-
 .../upgrades/upgrade_nonrolling_new_stack.xml   |   6 +-
 .../stacks/HDP/2.1.1/upgrades/upgrade_test.xml  |  45 +++-
 ambari-web/app/controllers/main/service/item.js |  25 ++-
 ambari-web/app/messages.js                      |   3 +-
 .../configs/config_recommendation_parser.js     |   9 +-
 ...onfig_with_override_recommendation_parser.js |   2 +-
 .../mixins/common/configs/enhanced_configs.js   |  13 +-
 ambari-web/app/templates/main/host/details.hbs  |   2 +-
 ambari-web/app/views/common/controls_view.js    |   7 +-
 .../app/views/common/quick_view_link_view.js    |   5 +-
 .../test/controllers/main/service/item_test.js  |  12 ++
 .../config_recommendation_parser_test.js        |  27 ++-
 .../test/views/common/quick_link_view_test.js   |  51 +++++
 78 files changed, 1359 insertions(+), 613 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-Derby-CREATE.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-SQLAnywhere-CREATE.sql
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/ambari/blob/4b36397f/ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql
----------------------------------------------------------------------


[04/19] ambari git commit: AMBARI-14711. Message displayed on hover on the host status indicator is half visible (alexantonenko)

Posted by nc...@apache.org.
AMBARI-14711. Message displayed on hover on the host status indicator is half visible (alexantonenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/36af9bba
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/36af9bba
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/36af9bba

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 36af9bba684278f40c93eb7e80687f9bcec8f86f
Parents: 8684e45
Author: Alex Antonenko <hi...@gmail.com>
Authored: Mon Jan 18 19:26:34 2016 +0200
Committer: Alex Antonenko <hi...@gmail.com>
Committed: Mon Jan 18 19:33:05 2016 +0200

----------------------------------------------------------------------
 ambari-web/app/templates/main/host/details.hbs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/36af9bba/ambari-web/app/templates/main/host/details.hbs
----------------------------------------------------------------------
diff --git a/ambari-web/app/templates/main/host/details.hbs b/ambari-web/app/templates/main/host/details.hbs
index eadf8a3..f709612 100644
--- a/ambari-web/app/templates/main/host/details.hbs
+++ b/ambari-web/app/templates/main/host/details.hbs
@@ -20,7 +20,7 @@
   <div id="host-details">
     <div class="host-details-header">
       <div class="status-info">
-        <span rel="HealthTooltip" {{bindAttr class="view.content.healthClass view.content.healthIconClass"}} {{bindAttr data-original-title="view.content.healthToolTip" }}></span><span
+        <span rel="HealthTooltip" {{bindAttr class="view.content.healthClass view.content.healthIconClass"}} data-placement="bottom" {{bindAttr data-original-title="view.content.healthToolTip" }}></span><span
               class='host-title'>{{unbound view.content.hostName}}</span>
         {{#unless view.isActive}}
           <span class="host-maintenance-notice pull-right"><span class="icon-medkit"></span> {{t hosts.host.passive.mode}}</span>