You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2016/04/23 00:38:25 UTC
ambari git commit: AMBARI-16031. Create "/hadoop/llap/local" on each
host and disk in Kerberized cluster for LLAP (alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk 9906c393a -> 4ddaed0eb
AMBARI-16031. Create "/hadoop/llap/local" on each host and disk in Kerberized cluster for LLAP (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4ddaed0e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4ddaed0e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4ddaed0e
Branch: refs/heads/trunk
Commit: 4ddaed0eb7ae617c85b7ddd97b84e3d47f4194bd
Parents: 9906c39
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Tue Apr 19 18:31:51 2016 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Fri Apr 22 15:23:43 2016 -0700
----------------------------------------------------------------------
.../HIVE/0.12.0.2.0/kerberos.json | 5 ++
.../common-services/YARN/2.1.0.2.0/metainfo.xml | 10 ++++
.../2.1.0.2.0/package/scripts/nodemanager.py | 12 ++++-
.../2.1.0.2.0/package/scripts/params_linux.py | 37 +++++++++++++--
.../YARN/2.1.0.2.0/package/scripts/yarn.py | 49 +++++++++++++++++++-
.../stacks/HDP/2.5/services/HIVE/kerberos.json | 3 +-
ambari-web/app/controllers/main/service/item.js | 38 +++++++++++++++
ambari-web/app/messages.js | 7 ++-
ambari-web/app/models/host_component.js | 7 +++
ambari-web/app/views/main/service/item.js | 13 +++++-
ambari-web/test/views/main/service/item_test.js | 2 +
11 files changed, 173 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
index 872bfac..a9e3840 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/kerberos.json
@@ -18,6 +18,11 @@
}
},
{
+ "hive-interactive-site": {
+ "hive.llap.daemon.work.dirs": "/hadoop/llap/local"
+ }
+ },
+ {
"webhcat-site": {
"templeton.kerberos.secret": "secret",
"templeton.hive.properties": "hive.metastore.local=false,hive.metastore.uris=${clusterHostInfo/hive_metastore_host|each(thrift://%s:9083, \\\\,, \\s*\\,\\s*)},hive.metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=hive/_HOST@${realm}"
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
index 10e6405..7034404 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/metainfo.xml
@@ -91,6 +91,16 @@
<!-- Used by decommission and recommission -->
<masterComponent>RESOURCEMANAGER</masterComponent>
</bulkCommands>
+ <customCommands>
+ <customCommand>
+ <name>CREATE_YARN_DIRECTORIES</name>
+ <commandScript>
+ <script>scripts/nodemanager.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </customCommand>
+ </customCommands>
<logs>
<log>
<logId>yarn_nodemanager</logId>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
index 6837c27..b1d6351 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
@@ -30,7 +30,7 @@ from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.security_commons import build_expectations, \
cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
FILE_TYPE_XML
-from yarn import yarn
+from yarn import yarn, create_hive_llap_work_dir
from service import service
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyImpl
@@ -90,6 +90,16 @@ class NodemanagerDefault(Nodemanager):
env.set_params(status_params)
check_process_status(status_params.nodemanager_pid_file)
+ def create_yarn_directories(self, env):
+ """
+ Custom command to Create Directories, e.g., needed by YARN Apps on Slider
+ """
+ Logger.info("Custom Command to Create Directories")
+ import params
+ env.set_params(params)
+ create_hive_llap_work_dir(params)
+
+
def security_status(self, env):
import status_params
env.set_params(status_params)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 19dbb60..8fbefb2 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -19,6 +19,7 @@ Ambari Agent
"""
import os
+import re
from resource_management.libraries.script.script import Script
from resource_management.libraries.resources.hdfs_resource import HdfsResource
@@ -184,11 +185,39 @@ nm_address = config['configurations']['yarn-site']['yarn.nodemanager.address']
if hostname and nm_address and nm_address.startswith("0.0.0.0:"):
nm_address = nm_address.replace("0.0.0.0", hostname)
-nm_local_dirs = config['configurations']['yarn-site']['yarn.nodemanager.local-dirs']
-nm_log_dirs = config['configurations']['yarn-site']['yarn.nodemanager.log-dirs']
+# Initialize lists of work directories.
+nm_local_dirs = default("/configurations/yarn-site/yarn.nodemanager.local-dirs", "")
+nm_log_dirs = default("/configurations/yarn-site/yarn.nodemanager.log-dirs", "")
-nm_local_dirs_list = nm_local_dirs.split(',')
-nm_log_dirs_list = nm_log_dirs.split(',')
+# If Hive Interactive Server is present, then default value will be ${yarn.nodemanager.local-dirs}
+# During kerberization, kerberos.json changes the value to "/hadoop/llap/local".
+# This is a comma separated list of directories.
+hive_llap_work_dirs = default("/configurations/hive-interactive-site/hive.llap.daemon.work.dirs", "")
+
+# It can also be a dynamic variable, such as "${yarn.nodemanager.local-dirs}"
+p = re.compile(r"\$\{(.*?)\}")
+m = p.search(hive_llap_work_dirs)
+if m and len(m.groups()) == 1:
+ hive_llap_work_dirs = default("/configurations/yarn-site/" + m.group(1), "")
+
+nm_local_dirs_list = nm_local_dirs.split(",")
+nm_log_dirs_list = nm_log_dirs.split(",")
+hive_llap_work_dirs_list = hive_llap_work_dirs.split(",")
+
+
+def cleanse_list(l):
+ if l is None:
+ return None
+ l = [e.strip() for e in l if e.strip() != ""]
+ return l if len(l) > 0 else None
+
+nm_local_dirs_list = cleanse_list(nm_local_dirs_list)
+nm_log_dirs_list = cleanse_list(nm_log_dirs_list)
+hive_llap_work_dirs_list = cleanse_list(hive_llap_work_dirs_list)
+
+# Needed for LLAP to determine if should create Hive LLAP Work Dirs.
+hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', [])
+hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts) > 0 else None
distrAppJarName = "hadoop-yarn-applications-distributedshell-2.*.jar"
hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index 735d8cb..5d1be15 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -19,11 +19,25 @@ Ambari Agent
"""
-from resource_management import *
+# Python Imports
import os
+
+# Ambari Common and Resource Management Imports
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources.service import ServiceConfig
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+from resource_management.core.logger import Logger
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
+# Local Imports
+
@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
def yarn(name = None):
@@ -113,6 +127,7 @@ def yarn(name = None):
recursive_ownership = True,
)
+ #<editor-fold desc="Node Manager Section">
if name == "nodemanager":
# First start after enabling/disabling security
@@ -156,6 +171,9 @@ def yarn(name = None):
recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'},
)
+ create_hive_llap_work_dir(params)
+ #</editor-fold>
+
if params.yarn_nodemanager_recovery_dir:
Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
@@ -445,3 +463,32 @@ def yarn(name = None):
owner=params.mapred_user,
group=params.user_group
)
+
+
+def create_hive_llap_work_dir(params):
+ """
+ Create the work directory needed by LLAP, which is required on all NodeManagers.
+ This needs to be whenever NodeManagers are restarted, or after Hive Server Interactive and LLAP are added and started
+ via a custom command
+ :param params: Command parameters dictionary.
+ """
+ if params.hive_llap_work_dirs_list is None or params.hive_server_interactive_host is None or not params.security_enabled:
+ Logger.info("Skip creating any Hive LLAP work dir since either it's empty, Hive Interactive is not present, "
+ "or cluster is not kerberized.")
+ return
+
+ # If we already created this dir list because it has the same value as NM Local Dirs, then skip it
+ skip = False
+ if params.nm_local_dirs_list is not None and set(params.nm_local_dirs_list) == set(params.hive_llap_work_dirs_list):
+ skip = True
+
+ if skip:
+ Logger.info(format("Skip creating Hive LLAP Work Dirs since it is equivalent to NM Local Dirs: {nm_local_dirs_list}"))
+ else:
+ Logger.info(format("Hive Server Interactive is present on the cluster, ensure that the Hive LLAP work dirs exist: {hive_llap_work_dirs_list}"))
+ Directory(params.hive_llap_work_dirs_list,
+ owner=params.yarn_user,
+ group=params.user_group,
+ create_parents=True,
+ ignore_failures=False,
+ mode=0775)
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
index a03a1b7..b5ec1c9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/kerberos.json
@@ -31,7 +31,8 @@
{
"hive-interactive-site": {
"hadoop.security.authentication": "KERBEROS",
- "hadoop.security.authorization": "true"
+ "hadoop.security.authorization": "true",
+ "hive.llap.daemon.work.dirs": "/hadoop/llap/local"
}
}
],
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/app/controllers/main/service/item.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/item.js b/ambari-web/app/controllers/main/service/item.js
index 5707882..bfede8a 100644
--- a/ambari-web/app/controllers/main/service/item.js
+++ b/ambari-web/app/controllers/main/service/item.js
@@ -635,6 +635,44 @@ App.MainServiceItemController = Em.Controller.extend(App.SupportClientConfigsDow
App.showAlertPopup(Em.I18n.t('common.error'), error);
},
+ createYARNDirectories: function(event) {
+ var context = Em.I18n.t('services.service.actions.run.createYARNDirectories');
+ var command = "CREATE_YARN_DIRECTORIES";
+ var component = "NODEMANAGER";
+ var controller = this;
+ var hosts = App.Service.find('YARN').get('hostComponents').filterProperty('componentName', component).mapProperty('hostName');
+ return App.showConfirmationPopup(function() {
+ App.ajax.send({
+ name: 'service.item.executeCustomCommand',
+ sender: controller,
+ data: {
+ command: command,
+ context: context,
+ hosts: hosts.join(','),
+ serviceName: "YARN",
+ componentName: component
+ },
+ success: 'createYARNDirectoriesSuccessCallback',
+ error: 'createYARNDirectoriesErrorCallback'
+ });
+ }, Em.I18n.t('services.service.actions.run.createYARNDirectories.confirmation'), null, Em.I18n.t('popup.confirmation.commonHeader'), Em.I18n.t('ok'), false);
+ },
+ createYARNDirectoriesSuccessCallback : function(data, ajaxOptions, params) {
+ if (data.Requests.id) {
+ App.router.get('backgroundOperationsController').showPopup();
+ }
+ },
+ createYARNDirectoriesErrorCallback : function(data) {
+ var error = Em.I18n.t('services.service.actions.run.executeCustomCommand.error');
+ if (data && data.responseText) {
+ try {
+ var json = $.parseJSON(data.responseText);
+ error += json.message;
+ } catch (err) {}
+ }
+ App.showAlertPopup(Em.I18n.t('common.error'), error);
+ },
+
/**
* On click handler for rebalance Hdfs command from items menu
*/
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/app/messages.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/messages.js b/ambari-web/app/messages.js
index 6c0049d..907d9ae 100644
--- a/ambari-web/app/messages.js
+++ b/ambari-web/app/messages.js
@@ -1752,9 +1752,12 @@ Em.I18n.translations = {
'services.service.actions.run.stopLdapKnox.context':'Stop Demo LDAP',
'services.service.actions.run.startStopLdapKnox.error': 'Error during remote command: ',
- // LLAP Custom Command
+ // Hive Server Interactive custom command to restart LLAP
'services.service.actions.run.restartLLAP':'Restart LLAP',
-
+ // Node Manager custom command to refresh YARN Apps on Slider
+ 'services.service.actions.run.createYARNDirectories':'Create YARN Directories',
+ 'services.service.actions.run.createYARNDirectories.confirmation':'Please confirm. This will create necessary folders on the NodeManagers required for running YARN Apps on Slider, such as LLAP. This does not need to restart NodeManagers.',
+
'services.service.actions.run.immediateStopHawqService.context':'Stop HAWQ Service (Immediate Mode)',
'services.service.actions.run.immediateStopHawqService.label':'Stop HAWQ Service (Immediate Mode)',
'services.service.actions.run.immediateStopHawqSegment.label':'Stop (Immediate Mode)',
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/app/models/host_component.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/models/host_component.js b/ambari-web/app/models/host_component.js
index b4a369e..f10947f 100644
--- a/ambari-web/app/models/host_component.js
+++ b/ambari-web/app/models/host_component.js
@@ -370,6 +370,13 @@ App.HostComponentActionMap = {
label: Em.I18n.t('services.service.actions.run.restartLLAP') + ' ∞',
cssClass: 'icon-refresh'
},
+ CREATE_YARN_DIRECTORIES: {
+ action: 'createYARNDirectories',
+ customCommand: 'CREATE_YARN_DIRECTORIES',
+ context: Em.I18n.t('services.service.actions.run.createYARNDirectories'),
+ label: Em.I18n.t('services.service.actions.run.createYARNDirectories'),
+ cssClass: 'icon-refresh'
+ },
REBALANCEHDFS: {
action: 'rebalanceHdfsNodes',
customCommand: 'REBALANCEHDFS',
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/app/views/main/service/item.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/views/main/service/item.js b/ambari-web/app/views/main/service/item.js
index 51f9bdd..36b3413 100644
--- a/ambari-web/app/views/main/service/item.js
+++ b/ambari-web/app/views/main/service/item.js
@@ -40,7 +40,8 @@ App.MainServiceItemView = Em.View.extend({
'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE', 'REMOVE_HAWQ_STANDBY', 'RESYNC_HAWQ_STANDBY'],
'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'],
'HAWQSTANDBY': ['ACTIVATE_HAWQ_STANDBY'],
- 'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"]
+ 'HIVE_SERVER_INTERACTIVE' : ["RESTART_LLAP"],
+ 'NODEMANAGER' : ["CREATE_YARN_DIRECTORIES"]
},
addActionMap: function() {
@@ -141,6 +142,16 @@ App.MainServiceItemView = Em.View.extend({
}
if (this.get('serviceName') === 'YARN') {
options.push(actionMap.REFRESHQUEUES);
+ var nodeManagerComponent = App.StackServiceComponent.find().findProperty('componentName', 'NODEMANAGER');
+ var isNodeManagerPresent = allSlaves.contains('NODEMANAGER');
+ if (nodeManagerComponent && isNodeManagerPresent) {
+ var nodeManagerCustomCommands = nodeManagerComponent.get('customCommands');
+ nodeManagerCustomCommands.forEach(function (command) {
+ if (actionMap[command]) {
+ options.push(actionMap[command]);
+ }
+ });
+ }
}
options.push(actionMap.RESTART_ALL);
allSlaves.concat(allMasters).filter(function (_component) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/4ddaed0e/ambari-web/test/views/main/service/item_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/views/main/service/item_test.js b/ambari-web/test/views/main/service/item_test.js
index 085eaad..f612f2d 100644
--- a/ambari-web/test/views/main/service/item_test.js
+++ b/ambari-web/test/views/main/service/item_test.js
@@ -494,6 +494,8 @@ describe('App.MainServiceItemView', function () {
return Em.Object.create({ customCommands: ["DECOMMISSION", "REBALANCEHDFS"] });
case 'RESOURCEMANAGER':
return Em.Object.create({ customCommands: ["DECOMMISSION", "REFRESHQUEUES"] });
+ case 'NODEMANAGER':
+ return Em.Object.create({ customCommands: ["REFRESHQUEUES", "CREATE_YARN_DIRECTORIES"] });
case 'HBASE_MASTER':
return Em.Object.create({ customCommands: ["DECOMMISSION"] });
case 'KNOX_GATEWAY':