You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ji...@apache.org on 2024/09/07 01:44:38 UTC
(ambari) branch trunk updated: AMBARI-25971: Add yarn TimelineService V2 and Registrydns support (#3786)
This is an automated email from the ASF dual-hosted git repository.
jialiang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5aa5048fbb AMBARI-25971: Add yarn TimelineService V2 and Registrydns support (#3786)
5aa5048fbb is described below
commit 5aa5048fbb172441131ff3e10f5faae1dc500c2c
Author: jialiang <ji...@gmail.com>
AuthorDate: Sat Sep 7 09:44:33 2024 +0800
AMBARI-25971: Add yarn TimelineService V2 and Registrydns support (#3786)
---
.../BIGTOP/3.2.0/services/YARN/YARN_widgets.json | 61 +-
.../stacks/BIGTOP/3.2.0/services/YARN/alerts.json | 86 +-
.../YARN/configuration/container-executor.xml | 223 ++++
.../services/YARN/configuration/resource-types.xml | 47 +
.../3.2.0/services/YARN/configuration/yarn-env.xml | 323 ++++--
.../services/YARN/configuration/yarn-hbase-env.xml | 272 +++++
.../YARN/configuration/yarn-hbase-log4j.xml | 188 ++++
.../YARN/configuration/yarn-hbase-policy.xml | 53 +
.../YARN/configuration/yarn-hbase-site.xml | 634 +++++++++++
.../services/YARN/configuration/yarn-site.xml | 1181 ++++++++++++++++++--
.../BIGTOP/3.2.0/services/YARN/kerberos.json | 223 +++-
.../stacks/BIGTOP/3.2.0/services/YARN/metainfo.xml | 88 ++
.../YARN/package/alerts/alert_ats_hbase.py | 246 ++++
.../services/YARN/package/scripts/functions.py | 43 +
.../services/YARN/package/scripts/hbase_service.py | 167 +++
.../services/YARN/package/scripts/historyserver.py | 22 +
.../services/YARN/package/scripts/params_linux.py | 254 ++++-
.../3.2.0/services/YARN/package/scripts/service.py | 150 ++-
.../services/YARN/package/scripts/status_params.py | 29 +
.../{historyserver.py => timelinereader.py} | 86 +-
.../3.2.0/services/YARN/package/scripts/yarn.py | 245 +++-
.../{historyserver.py => yarn_registry_dns.py} | 58 +-
.../templates/hadoop-metrics2-hbase.properties.j2 | 62 +
.../package/templates/input.config-yarn.json.j2 | 86 ++
.../YARN/package/templates/yarn_ats_jaas.conf.j2 | 9 +
.../templates/yarn_hbase_grant_permissions.j2 | 39 +
...jaas.conf.j2 => yarn_hbase_master_jaas.conf.j2} | 25 +-
.../templates/yarn_hbase_package_preparation.j2 | 76 ++
...onf.j2 => yarn_hbase_regionserver_jaas.conf.j2} | 25 +-
.../templates/yarn_hbase_secure.yarnfile.j2 | 172 +++
.../templates/yarn_hbase_unsecure.yarnfile.j2 | 168 +++
.../YARN/package/templates/yarn_nm_jaas.conf.j2 | 9 +
...jaas.conf.j2 => yarn_registry_dns_jaas.conf.j2} | 14 +-
.../YARN/properties/container-executor.cfg.j2 | 19 +
.../3.2.0/services/YARN/quicklinks/quicklinks.json | 2 +-
.../3.2.0/services/YARN/role_command_order.json | 4 +-
36 files changed, 5078 insertions(+), 311 deletions(-)
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/YARN_widgets.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/YARN_widgets.json
index bef0dc03fe..ac91ad0569 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/YARN_widgets.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/YARN_widgets.json
@@ -71,6 +71,35 @@
"time_range": "1"
}
},
+ {
+ "widget_name": "Bad Local Disks",
+ "description": "Number of unhealthy local disks across all NodeManagers.",
+ "widget_type": "NUMBER",
+ "is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.BadLocalDirs",
+ "metric_path": "metrics/yarn/BadLocalDirs",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.BadLogDirs",
+ "metric_path": "metrics/yarn/BadLogDirs",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "Number of unhealthy local disks for NodeManager",
+ "value": "${yarn.NodeManagerMetrics.BadLocalDirs + yarn.NodeManagerMetrics.BadLogDirs}"
+ }
+ ],
+ "properties": {
+ "display_unit": ""
+ }
+ },
{
"widget_name": "Container Failures",
"description": "Percentage of all containers failing in the cluster.",
@@ -372,10 +401,40 @@
"section_name": "YARN_HEATMAPS",
"widgetLayoutInfo": [
{
- "widget_name": "Total Allocatable RAM Utilized per NodeManager",
+ "widget_name": "YARN local disk space utilization per NodeManager",
"description": "",
"widget_type": "HEATMAP",
"is_visible": true,
+ "metrics": [
+ {
+ "name": "yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc",
+ "metric_path": "metrics/yarn/GoodLocalDirsDiskUtilizationPerc",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ },
+ {
+ "name": "yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc",
+ "metric_path": "metrics/yarn/GoodLogDirsDiskUtilizationPerc",
+ "service_name": "YARN",
+ "component_name": "NODEMANAGER"
+ }
+ ],
+ "values": [
+ {
+ "name": "YARN local disk space utilization per NodeManager",
+ "value": "${(yarn.NodeManagerMetrics.GoodLocalDirsDiskUtilizationPerc + yarn.NodeManagerMetrics.GoodLogDirsDiskUtilizationPerc)/2}"
+ }
+ ],
+ "properties": {
+ "display_unit": "%",
+ "max_limit": "100"
+ }
+ },
+ {
+ "widget_name": "Total Allocatable RAM Utilized per NodeManager",
+ "description": "",
+ "widget_type": "HEATMAP",
+ "is_visible": false,
"metrics": [
{
"name": "yarn.NodeManagerMetrics.AllocatedGB",
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/alerts.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/alerts.json
index f2962dc6f1..08b5e131e7 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/alerts.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/alerts.json
@@ -99,7 +99,7 @@
"warning": {
"text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
"value": 3000
- },
+ },
"critical": {
"text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
"value": 5000
@@ -316,7 +316,7 @@
"warning": {
"text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
"value": 3000
- },
+ },
"critical": {
"text": "Average Queue Time:[{0}], Average Processing Time:[{1}]",
"value": 5000
@@ -387,6 +387,88 @@
}
}
}
+ ],
+ "YARN_REGISTRY_DNS": [
+ {
+ "name": "YARN_REGISTRY_DNS_PROCESS",
+ "label": "Registry DNS",
+ "description": "This host-level alert is triggered if the Registry DNS Service cannot be determined to be up and listening on the network.",
+ "interval": 1,
+ "scope": "HOST",
+ "source": {
+ "type": "PORT",
+ "uri": "{{yarn-env/registry.dns.bind-port}}",
+ "default_port": 53,
+ "reporting": {
+ "ok": {
+ "text": "TCP OK - {0:.3f}s response on port {1}"
+ },
+ "warning": {
+ "text": "TCP OK - {0:.3f}s response on port {1}",
+ "value": 1.5
+ },
+ "critical": {
+ "text": "Connection failed: {0} to {1}:{2}",
+ "value": 5
+ }
+ }
+ }
+ }
+ ],
+ "TIMELINE_READER": [
+ {
+ "name": "yarn_timeline_reader_webui",
+ "label": "Timeline Reader Web UI",
+ "description": "This host-level alert is triggered if Timeline Reader Web UI is unreachable.",
+ "interval": 1,
+ "scope": "ANY",
+ "source": {
+ "type": "WEB",
+ "uri": {
+ "http": "{{yarn-site/yarn.timeline-service.reader.webapp.address}}/ws/v2/timeline",
+ "https": "{{yarn-site/yarn.timeline-service.reader.webapp.https.address}}/ws/v2/timeline",
+ "https_property": "{{yarn-site/yarn.http.policy}}",
+ "https_property_value": "HTTPS_ONLY",
+ "kerberos_keytab": "{{cluster-env/smokeuser_keytab}}",
+ "kerberos_principal": "{{cluster-env/smokeuser_principal_name}}",
+ "connection_timeout": 5.0
+ },
+ "reporting": {
+ "ok": {
+ "text": "HTTP {0} response in {2:.3f}s"
+ },
+ "warning":{
+ "text": "HTTP {0} response from {1} in {2:.3f}s ({3})"
+ },
+ "critical": {
+ "text": "Connection failed to {1} ({3})"
+ }
+ }
+ }
+ },
+ {
+ "name": "ats_hbase",
+ "label": "ATSv2 HBase Application",
+ "description": "This alert is triggered if the ats hbase cannot be determined to be up and responding to requests.",
+ "interval": 5,
+ "scope": "ANY",
+ "enabled": true,
+ "source": {
+ "type": "SCRIPT",
+ "path": "BIGTOP/3.2.0/services/YARN/package/alerts/alert_ats_hbase.py",
+ "parameters": [
+ {
+ "name": "check.command.timeout",
+ "display_name": "Command Timeout",
+ "value": 120.0,
+ "type": "NUMERIC",
+ "description": "The maximum time before check command will be killed by timeout",
+ "units": "seconds",
+ "threshold": "CRITICAL"
+ }
+ ]
+ }
+ }
]
}
}
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/container-executor.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/container-executor.xml
index e19fe9049b..bce09f90eb 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/container-executor.xml
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/container-executor.xml
@@ -20,6 +20,229 @@
*/
-->
<configuration supports_adding_forbidden="true">
+ <property>
+ <name>docker_module_enabled</name>
+ <value>false</value>
+ <display-name>Docker Runtime</display-name>
+ <description>Need user to pre-install docker environment first.
+ </description>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>docker_binary</name>
+ <value>/usr/bin/docker</value>
+ <display-name>Docker Binary</display-name>
+ <description>The binary used to launch docker containers.</description>
+ <value-attributes>
+ <type>directory</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>docker_allowed_devices</name>
+ <value/>
+ <display-name>Docker Allowed Devices</display-name>
+ <description>Comma seperated list of devices that can be mounted into a
+ container.
+ </description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>docker_module_enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>docker_allowed_ro-mounts</name>
+ <value/>
+ <display-name>Docker Allowed Read-only Mounts</display-name>
+ <description>
+ Comma seperated volumes that can be mounted as read-only.
+ </description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>docker_module_enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>docker_allowed_rw-mounts</name>
+ <value/>
+ <display-name>Docker Allowed Read-write Mounts</display-name>
+ <description>
+ Comma seperate volumes that can be mounted as read-write. Add the
+ yarn
+ local and log dirs to this list to run Hadoop jobs.
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>docker_allowed_volume-drivers</name>
+ <value/>
+ <display-name>Docker Allowed Volume-drivers</display-name>
+ <description>
+ Comma seperated list of allowed volume-drivers.
+ </description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>docker_module_enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>docker_privileged-containers_enabled</name>
+ <value>false</value>
+ <display-name>Enable Launching Privileged Containers</display-name>
+ <description>
+ Whether to enable launching privileged docker containers.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>docker_trusted_registries</name>
+ <value></value>
+ <display-name>Docker Trusted Registries</display-name>
+ <description>
+ List of registries whose docker images are allowed to mount volumes and launch privileged docker containers.
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>min_user_id</name>
+ <value>1000</value>
+ <display-name>Minimum user ID for submitting job</display-name>
+ <description>Set to 0 to disallow root from submitting jobs. Set to 1000 to
+ disallow all superusers from submitting jobs</description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>gpu_module_enabled</name>
+ <value>false</value>
+ <display-name>GPU Scheduling and Isolation</display-name>
+ <description>Whether to enable gpu scheduling and isolation.
+ </description>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>cgroup_root</name>
+ <value>/sys/fs/cgroup</value>
+ <display-name>CGroup Root Path</display-name>
+ <description>This should be same as
+ yarn.nodemanager.linux-container-executor.cgroups.mount-path inside
+ yarn-site.xml</description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <type>directory</type>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>yarn_hierarchy</name>
+ <value>yarn</value>
+ <display-name>Yarn CGroup Hierarchy</display-name>
+ <description>This should be same as
+ yarn.nodemanager.linux-container-executor.cgroups.hierarchy
+ inside yarn-site.xml</description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>docker_module_enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
<property>
<name>content</name>
<display-name>container-executor configuration template</display-name>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/resource-types.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/resource-types.xml
new file mode 100755
index 0000000000..65e8254513
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/resource-types.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>yarn.resource-types</name>
+ <value></value>
+ <description>Enable resource types other than memory and vcores, values split by comma. For example value=yarn.io/gpu enables GPU resource types</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resource-types.yarn.io_gpu.maximum-allocation</name>
+ <value>8</value>
+ <display-name>Maximum Container Size (GPU)</display-name>
+ <description>Maximum GPU Allocation</description>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>8</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-env.xml
index 81ff757f7a..726bd83027 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-env.xml
@@ -59,7 +59,25 @@
</property>
</user-groups>
</value-attributes>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_ats_user</name>
+ <display-name>Yarn ATS User</display-name>
+ <value>yarn-ats</value>
+ <property-type>USER</property-type>
+ <description>YARN Client User for ATSv2 backend</description>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ <user-groups>
+ <property>
+ <type>cluster-env</type>
+ <name>user_group</name>
+ </property>
+ </user-groups>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>yarn_heapsize</name>
@@ -121,138 +139,232 @@
<name>yarn_user_nproc_limit</name>
<value>65536</value>
<description>Max number of processes limit setting for YARN user.</description>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- These properties were inherited from HDP 2.1 -->
+ <property>
+ <name>apptimelineserver_heapsize</name>
+ <value>1024</value>
+ <display-name>AppTimelineServer Java heap size</display-name>
+ <description>Max heapsize for AppTimelineServer using a numerical value in the scale of MB</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <unit>MB</unit>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- These properties were inherited from HDP 2.2 -->
+ <property>
+ <name>yarn_cgroups_enabled</name>
+ <value>false</value>
+ <description>You can use CGroups to isolate CPU-heavy processes in a Hadoop cluster.</description>
+ <display-name>CPU Scheduling and Isolation</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
</property>
+
+ <property>
+ <name>registry.dns.bind-port</name>
+ <display-name>RegistryDNS Bind Port</display-name>
+ <value>53</value>
+ <description>
+ The port number for the DNS listener. The default port is 53.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
<!-- yarn-env.sh -->
<property>
<name>content</name>
<display-name>yarn-env template</display-name>
<description>This is the jinja template for yarn-env.sh file</description>
<value>
-export HADOOP_YARN_HOME={{hadoop_yarn_home}}
-USER="$(whoami)"
-export HADOOP_LOG_DIR={{yarn_log_dir_prefix}}/$USER
-export HADOOP_PID_DIR={{yarn_pid_dir_prefix}}/$USER
-export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
-export JAVA_HOME={{java64_home}}
-export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
+ export HADOOP_YARN_HOME={{hadoop_yarn_home}}
+ USER="$(whoami)"
+ export HADOOP_LOG_DIR={{yarn_log_dir_prefix}}/$USER
+ export HADOOP_PID_DIR={{yarn_pid_dir_prefix}}/$USER
+ export HADOOP_SECURE_LOG_DIR={{yarn_log_dir_prefix}}/$USER
+ export HADOOP_SECURE_PID_DIR={{yarn_pid_dir_prefix}}/$USER
+ export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
+ export JAVA_HOME={{java64_home}}
+ export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
+
+ # We need to add the EWMA and RFA appender for the yarn daemons only;
+ # however, HADOOP_ROOT_LOGGER is shared by the yarn client and the
+ # daemons. This is restrict the EWMA appender to daemons only.
+ export HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO}
+ export HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-INFO,console}
+ export HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_DAEMON_ROOT_LOGGER:-${HADOOP_LOGLEVEL},EWMA,RFA}
+
+ # User for YARN daemons
+ export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+ # resolve links - $0 may be a softlink
+ export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}"
+ # some Java parameters
+ # export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+ if [ "$JAVA_HOME" != "" ]; then
+ #echo "run java in $JAVA_HOME"
+ JAVA_HOME=$JAVA_HOME
+ fi
-# User for YARN daemons
-export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
+ if [ "$JAVA_HOME" = "" ]; then
+ echo "Error: JAVA_HOME is not set."
+ exit 1
+ fi
-# resolve links - $0 may be a softlink
-export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_YARN_HOME/etc/hadoop}"
+ JAVA=$JAVA_HOME/bin/java
+ JAVA_HEAP_MAX=-Xmx1000m
-# some Java parameters
-# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
-if [ "$JAVA_HOME" != "" ]; then
- #echo "run java in $JAVA_HOME"
- JAVA_HOME=$JAVA_HOME
-fi
+ # For setting YARN specific HEAP sizes please use this
+ # Parameter and set appropriately
+ YARN_HEAPSIZE={{yarn_heapsize}}
-if [ "$JAVA_HOME" = "" ]; then
- echo "Error: JAVA_HOME is not set."
- exit 1
-fi
+ # check envvars which might override default args
+ if [ "$YARN_HEAPSIZE" != "" ]; then
+ JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
+ fi
-JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m
+ # Resource Manager specific parameters
-# For setting YARN specific HEAP sizes please use this
-# Parameter and set appropriately
-YARN_HEAPSIZE={{yarn_heapsize}}
+ # Specify the max Heapsize for the ResourceManager using a numerical value
+ # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+ # the value to 1000.
+ # This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+ # and/or YARN_RESOURCEMANAGER_OPTS.
+ # If not specified, the default value will be picked from either YARN_HEAPMAX
+ # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+ export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}
-# check envvars which might override default args
-if [ "$YARN_HEAPSIZE" != "" ]; then
- JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m"
-fi
+ # Specify the JVM options to be used when starting the ResourceManager.
+ # These options will be appended to the options specified as HADOOP_OPTS
+ # and therefore may override any similar flags set in HADOOP_OPTS
+ {% if security_enabled %}
+ export YARN_RESOURCEMANAGER_OPTS="-Djava.security.auth.login.config={{yarn_jaas_file}}"
+ {% endif %}
-# Resource Manager specific parameters
+ # Node Manager specific parameters
-# Specify the max Heapsize for the ResourceManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
-# and/or YARN_RESOURCEMANAGER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}
+ # Specify the max Heapsize for the NodeManager using a numerical value
+ # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+ # the value to 1000.
+ # This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+ # and/or YARN_NODEMANAGER_OPTS.
+ # If not specified, the default value will be picked from either YARN_HEAPMAX
+ # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+ export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}
-# Specify the JVM options to be used when starting the ResourceManager.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#export YARN_RESOURCEMANAGER_OPTS=
+ # Specify the max Heapsize for the timeline server using a numerical value
+ # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
+ # the value to 1024.
+ # This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
+ # and/or YARN_TIMELINESERVER_OPTS.
+ # If not specified, the default value will be picked from either YARN_HEAPMAX
+ # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
+ export YARN_TIMELINESERVER_HEAPSIZE={{apptimelineserver_heapsize}}
-# Node Manager specific parameters
+ {% if security_enabled %}
+ export YARN_TIMELINESERVER_OPTS="-Djava.security.auth.login.config={{yarn_ats_jaas_file}}"
+ {% endif %}
-# Specify the max Heapsize for the NodeManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1000.
-# This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
-# and/or YARN_NODEMANAGER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}
+ {% if security_enabled %}
+ export YARN_TIMELINEREADER_OPTS="-Djava.security.auth.login.config={{yarn_ats_jaas_file}}"
+ {% endif %}
-# Specify the max Heapsize for the HistoryManager using a numerical value
-# in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set
-# the value to 1024.
-# This value will be overridden by an Xmx setting specified in either HADOOP_OPTS
-# and/or YARN_HISTORYSERVER_OPTS.
-# If not specified, the default value will be picked from either YARN_HEAPMAX
-# or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.
-export YARN_HISTORYSERVER_HEAPSIZE={{apptimelineserver_heapsize}}
+ {% if security_enabled %}
+ export YARN_REGISTRYDNS_OPTS="-Djava.security.auth.login.config={{yarn_registry_dns_jaas_file}}"
+ {% endif %}
-# Specify the JVM options to be used when starting the NodeManager.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#export YARN_NODEMANAGER_OPTS=
+ # Specify the JVM options to be used when starting the NodeManager.
+ # These options will be appended to the options specified as HADOOP_OPTS
+ # and therefore may override any similar flags set in HADOOP_OPTS
+ {% if security_enabled %}
+ export YARN_NODEMANAGER_OPTS="-Djava.security.auth.login.config={{yarn_nm_jaas_file}} -Dsun.security.krb5.rcache=none"
+ {% endif %}
-# so that filenames w/ spaces are handled correctly in loops below
-IFS=
+ # so that filenames w/ spaces are handled correctly in loops below
+ IFS=
-# default log directory and file
-if [ "$HADOOP_LOG_DIR" = "" ]; then
- HADOOP_LOG_DIR="$HADOOP_YARN_HOME/logs"
-fi
-if [ "$HADOOP_LOGFILE" = "" ]; then
- HADOOP_LOGFILE='yarn.log'
-fi
+ # default log directory and file
+ if [ "$HADOOP_LOG_DIR" = "" ]; then
+ HADOOP_LOG_DIR="$HADOOP_YARN_HOME/logs"
+ fi
+ if [ "$HADOOP_LOGFILE" = "" ]; then
+ HADOOP_LOGFILE='yarn.log'
+ fi
-# default policy file for service-level authorization
-if [ "$YARN_POLICYFILE" = "" ]; then
- YARN_POLICYFILE="hadoop-policy.xml"
-fi
+ # default policy file for service-level authorization
+ if [ "$YARN_POLICYFILE" = "" ]; then
+ YARN_POLICYFILE="hadoop-policy.xml"
+ fi
-# restore ordinary behaviour
-unset IFS
+ # restore ordinary behaviour
+ unset IFS
+ # YARN now uses specific subcommand options of the pattern (command)_(subcommand)_OPTS for every
+ # component. Because of this, HADDOP_OPTS is now used as a simple way to specify common properties
+ # between all YARN components.
+ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
+ HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.dir=$HADOOP_LOG_DIR"
+ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
+ HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.file=$HADOOP_LOGFILE"
+ HADOOP_OPTS="$HADOOP_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
+ HADOOP_OPTS="$HADOOP_OPTS -Dyarn.id.str=$HADOOP_IDENT_STRING"
+ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+ HADOOP_OPTS="$HADOOP_OPTS -Dyarn.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+ export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT"
+ export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT"
+ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
+ HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
+ fi
+ HADOOP_OPTS="$HADOOP_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+ HADOOP_OPTS="$HADOOP_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
-HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
-HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.dir=$HADOOP_LOG_DIR"
-HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
-HADOOP_OPTS="$HADOOP_OPTS -Dyarn.log.file=$HADOOP_LOGFILE"
-HADOOP_OPTS="$HADOOP_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
-HADOOP_OPTS="$HADOOP_OPTS -Dyarn.id.str=$HADOOP_IDENT_STRING"
-HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
-HADOOP_OPTS="$HADOOP_OPTS -Dyarn.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
-export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT"
-export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT"
-if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
- HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
-fi
-HADOOP_OPTS="$HADOOP_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
-HADOOP_OPTS="$HADOOP_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
+ {% if security_enabled %}
+ HADOOP_OPTS="$HADOOP_OPTS -Djavax.security.auth.useSubjectCredsOnly=false"
+ {% endif %}
-{% if rm_security_opts is defined %}
-HADOOP_OPTS="{{rm_security_opts}} $HADOOP_OPTS"
-{% endif %}
+ {% if rm_security_opts is defined %}
+ YARN_RESOURCEMANAGER_OPTS="{{rm_security_opts}} $YARN_RESOURCEMANAGER_OPTS"
+ {% endif %}
+
+ export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT"
+ export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS -Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY -Drm.audit.logger=INFO,RMAUDIT"
+
+ {% if registry_dns_needs_privileged_access %}
+ # If the DNS server is configured to use the standard privileged port 53,
+ # the environment variables YARN_REGISTRYDNS_SECURE_USER and
+ # YARN_REGISTRYDNS_SECURE_EXTRA_OPTS must be set.
+ export YARN_REGISTRYDNS_SECURE_USER={{yarn_user}}
+ export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"
+ {% endif %}
</value>
<value-attributes>
<type>content</type>
</value-attributes>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>service_check.queue.name</name>
@@ -268,4 +380,15 @@ HADOOP_OPTS="{{rm_security_opts}} $HADOOP_OPTS"
</depends-on>
<on-ambari-upgrade add="false"/>
</property>
+ <property>
+ <name>yarn_ats_principal_name</name>
+ <description>Yarn Client principal name</description>
+ <property-type>KERBEROS_PRINCIPAL</property-type>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_ats_user_keytab</name>
+ <description>Yarn Client keytab path</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
</configuration>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-env.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-env.xml
new file mode 100755
index 0000000000..66fe45c766
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-env.xml
@@ -0,0 +1,272 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+
+ <property>
+ <name>yarn_hbase_pid_dir_prefix</name>
+ <value>/var/run/hadoop-yarn-hbase</value>
+ <display-name>HBase PID Dir</display-name>
+ <description>Pid Directory for HBase.</description>
+ <value-attributes>
+ <type>directory</type>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase_java_io_tmpdir</name>
+ <value>/tmp</value>
+ <description>Used in hbase-env.sh as HBASE_OPTS=-Djava.io.tmpdir=java_io_tmpdir</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <!-- hbase-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>hbase-env template</display-name>
+ <description>This is the jinja template for hbase-env.sh file</description>
+ <value>
+ # Set environment variables here.
+
+ # The java implementation to use. Java 1.6 required.
+ export JAVA_HOME={{java64_home}}
+
+ # HBase Configuration directory
+ export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{yarn_hbase_conf_dir}}}
+
+ # Extra Java CLASSPATH elements. Optional.
+ export HBASE_CLASSPATH=${HBASE_CLASSPATH}
+
+
+ # The maximum amount of heap to use. Default is left to JVM default.
+ # export HBASE_HEAPSIZE=4G
+
+ # Extra Java runtime options.
+ # Below are what we set by default. May only work with SUN JVM.
+ # For more on why as well as other possible settings,
+ # see http://wiki.apache.org/hadoop/PerformanceTuning
+ export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{yarn_hbase_log_dir}}/gc.log-`date +'%Y%m%d%H%M'`"
+ # Uncomment below to enable java garbage collection logging.
+ # export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
+ # Uncomment and adjust to enable JMX exporting
+ # See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+ # More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+ #
+ # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+ # If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size
+ # export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+ # export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+ # File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+ export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
+
+ # Extra ssh options. Empty by default.
+ # export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+ # Where log files are stored. $HBASE_HOME/logs by default.
+ export HBASE_LOG_DIR=${HBASE_LOG_DIR:-{{yarn_hbase_log_dir}}}
+
+ # A string representing this instance of hbase. $USER by default.
+ # export HBASE_IDENT_STRING=$USER
+
+ # The scheduling priority for daemon processes. See 'man nice'.
+ # export HBASE_NICENESS=10
+
+ # The directory where pid files are stored. /tmp by default.
+ export HBASE_PID_DIR=${HBASE_PID_DIR:-{{yarn_hbase_pid_dir}}}
+
+ # Seconds to sleep between slave commands. Unset by default. This
+ # can be useful in large clusters, where, e.g., slave rsyncs can
+ # otherwise arrive faster than the master can service them.
+ # export HBASE_SLAVE_SLEEP=0.1
+
+ # Tell HBase whether it should manage it's own instance of Zookeeper or not.
+ export HBASE_MANAGES_ZK=false
+
+ {% if java_version < 8 %}
+ JDK_DEPENDED_OPTS="-XX:PermSize=128m -XX:MaxPermSize=128m"
+ {% endif %}
+
+ export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile=$HBASE_LOG_DIR/hs_err_pid%p.log -Djava.io.tmpdir={{yarn_hbase_java_io_tmpdir}}"
+ export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xmx{{yarn_hbase_master_heapsize}} $JDK_DEPENDED_OPTS"
+ export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -XX:CMSInitiatingOccupancyFraction=70 -XX:ReservedCodeCacheSize=256m -Xms{{yarn_hbase_regionserver_heapsize}} -Xmx{{yarn_hbase_regionserver_heapsize}} $JDK_DEPENDED_OPTS"
+
+ {% if security_enabled %}
+ export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{yarn_hbase_master_jaas_file}}"
+ export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{yarn_hbase_regionserver_jaas_file}}"
+ {% endif %}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>is_hbase_system_service_launch</name>
+ <value>false</value>
+ <description>Should Hbase cluster started as system service. This
+ configuration depends on use_external_hbase property. If
+ use_external_hbase is set, then this property doesn't take effect.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_system_service_queue_name</name>
+ <value>default</value>
+ <description>
+ The queue that used by service check.
+ </description>
+ <depends-on>
+ <property>
+ <type>capacity-scheduler</type>
+ <name>yarn.scheduler.capacity.root.queues</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_system_service_launch_mode</name>
+ <value>sync</value>
+ <description>Should services are launched as sync or async.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>yarn_hbase_master_cpu</name>
+ <value>1</value>
+ <description>Number of CPU for master container.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_master_memory</name>
+ <value>4096</value>
+ <description>master container memory in MB.</description>
+ <value-attributes>
+ <type>int</type>
+ <minimum>2048</minimum>
+ <maximum>4096</maximum>
+ <unit>MB</unit>
+ <increment-step>256</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_master_containers</name>
+ <value>1</value>
+ <description>Number of containers to launch for master.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>yarn_hbase_regionserver_cpu</name>
+ <value>1</value>
+ <description>Number of CPU for regionserver container.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_regionserver_memory</name>
+ <value>4096</value>
+ <description>regionserver container memory in MB.</description>
+ <value-attributes>
+ <type>int</type>
+ <minimum>2048</minimum>
+ <maximum>4096</maximum>
+ <unit>MB</unit>
+ <increment-step>256</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_regionserver_containers</name>
+ <value>1</value>
+ <description>Number of containers to launch for regionserver.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>yarn_hbase_client_cpu</name>
+ <value>1</value>
+ <description>Number of CPU for client container.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_client_memory</name>
+ <value>1536</value>
+ <description>client container memory in MB.</description>
+ <value-attributes>
+ <type>int</type>
+ <minimum>1024</minimum>
+ <maximum>2048</maximum>
+ <unit>MB</unit>
+ <increment-step>256</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_client_containers</name>
+ <value>1</value>
+ <description>Number of containers to launch for client.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn_hbase_heap_memory_factor</name>
+ <value>0.8</value>
+ <description>Heap memory is auto derived using this factor.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>use_external_hbase</name>
+ <value>false</value>
+ <description>Setting true, doesn't start embedded hbase or system service
+ hbase. Note: Admin/User need to take care of pointing right hbase-site.xml
+ into RM/NM classpath. If system service hbase is started, then admin must
+ clean up system service hbase before making this change.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>hbase_within_cluster</name>
+ <value>false</value>
+ <description>Set to true, YARN will use the HBase installed on this cluster by Ambari for storage.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <property>
+ <name>yarn_hbase_log_level</name>
+ <value>INFO</value>
+ <description>Setting log level to hmaster and regionserver. Default to info logs. Log levels could be INFO, DEBUG, WARN</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-log4j.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-log4j.xml
new file mode 100755
index 0000000000..ee3dcce232
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-log4j.xml
@@ -0,0 +1,188 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="false">
+ <property>
+ <name>hbase_log_maxfilesize</name>
+ <value>256</value>
+ <description>The maximum size of backup file before the log is rotated</description>
+ <display-name>HBase Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase_log_maxbackupindex</name>
+ <value>20</value>
+ <description>The number of backup files</description>
+ <display-name>HBase Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase_security_log_maxfilesize</name>
+ <value>256</value>
+ <description>The maximum size of security backup file before the log is rotated</description>
+ <display-name>HBase Security Log: backup file size</display-name>
+ <value-attributes>
+ <unit>MB</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase_security_log_maxbackupindex</name>
+ <value>20</value>
+ <description>The number of security backup files</description>
+ <display-name>HBase Security Log: # of backup files</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>content</name>
+ <display-name>hbase-log4j template</display-name>
+ <description>Custom log4j.properties</description>
+ <value>
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+
+ # Define some default values that can be overridden by system properties
+ hbase.root.logger=INFO,console
+ hbase.security.logger=INFO,console
+ hbase.log.dir=.
+ hbase.log.file=hbase.log
+
+ # Define the root logger to the system property "hbase.root.logger".
+ log4j.rootLogger=${hbase.root.logger}
+
+ # Logging Threshold
+ log4j.threshold=ALL
+
+ #
+ # Daily Rolling File Appender
+ #
+ log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+ log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+ # Rollver at midnight
+ log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+ # 30-day backup
+ #log4j.appender.DRFA.MaxBackupIndex=30
+ log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+ # Pattern format: Date LogLevel LoggerName LogMessage
+ log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+ # Rolling File Appender properties
+ hbase.log.maxfilesize={{hbase_log_maxfilesize}}MB
+ hbase.log.maxbackupindex={{hbase_log_maxbackupindex}}
+
+ # Rolling File Appender
+ log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+ log4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}
+
+ log4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}
+ log4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex}
+
+ log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+ log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+ #
+ # Security audit appender
+ #
+ hbase.security.log.file=SecurityAuth.audit
+ hbase.security.log.maxfilesize={{hbase_security_log_maxfilesize}}MB
+ hbase.security.log.maxbackupindex={{hbase_security_log_maxbackupindex}}
+ log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
+ log4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}
+ log4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}
+ log4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}
+ log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
+ log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+ log4j.category.SecurityLogger=${hbase.security.logger}
+ log4j.additivity.SecurityLogger=false
+ #log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE
+
+ #
+ # Null Appender
+ #
+ log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+ #
+ # console
+ # Add "console" to rootlogger above if you want to use this
+ #
+ log4j.appender.console=org.apache.log4j.ConsoleAppender
+ log4j.appender.console.target=System.err
+ log4j.appender.console.layout=org.apache.log4j.PatternLayout
+ log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+ # Custom Logging levels
+
+ log4j.logger.org.apache.zookeeper=INFO
+ #log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+ log4j.logger.org.apache.hadoop.hbase=INFO
+ # Make these two classes INFO-level. Make them DEBUG to see more zk debug.
+ log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO
+ log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO
+ #log4j.logger.org.apache.hadoop.dfs=DEBUG
+ # Set this class to log INFO only otherwise its OTT
+ # Enable this to get detailed connection error/retry logging.
+ # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE
+
+
+ # Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)
+ #log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG
+
+ # Uncomment the below if you want to remove logging of client region caching'
+ # and scan of .META. messages
+ # log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO
+ # log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO
+
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-policy.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-policy.xml
new file mode 100755
index 0000000000..548352b822
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-policy.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HRegionInterface protocol implementations (ie.
+ clients talking to HRegionServers)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>security.admin.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterInterface protocol implementation (ie.
+ clients talking to HMaster for admin operations).
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>security.masterregion.protocol.acl</name>
+ <value>*</value>
+ <description>ACL for HMasterRegionInterface protocol implementations
+ (for HRegionServers communicating with HMaster)
+ The ACL is a comma-separated list of user and group names. The user and
+ group list is separated by a blank. For e.g. "alice,bob users,wheel".
+ A special value of "*" means all users are allowed.</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+</configuration>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-site.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-site.xml
new file mode 100755
index 0000000000..d6a34f8de8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-hbase-site.xml
@@ -0,0 +1,634 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+
+ <property>
+ <name>hbase.rootdir</name>
+ <value>/atsv2/hbase/data</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ <on-ambari-upgrade add="false"/>
+ <description>The mode the cluster will be in. Possible values are false for
+ standalone mode and true for distributed mode. If false, startup will run
+ all HBase and ZooKeeper daemons together in the one JVM.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.port</name>
+ <value>17000</value>
+ <display-name>HBase Master Port</display-name>
+ <value-attributes>
+ <type>int</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.tmp.dir</name>
+ <value>/tmp/hbase-${user.name}</value>
+ <display-name>HBase tmp directory</display-name>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.local.dir</name>
+ <value>${hbase.tmp.dir}/local</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.master.info.bindAddress</name>
+ <value>0.0.0.0</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.master.info.port</name>
+ <value>17010</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.regionserver.info.port</name>
+ <value>17030</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>zookeeper.session.timeout</name>
+ <value>90000</value>
+ <display-name>Zookeeper Session Timeout</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>10000</minimum>
+ <maximum>180000</maximum>
+ <unit>milliseconds</unit>
+ <increment-step>10000</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.client.retries.number</name>
+ <value>7</value>
+ <description>Maximum retries. Used as maximum for all retryable
+ operations such as the getting of a cell's value, starting a row update,
+ etc. Retry interval is a rough function based on hbase.client.pause. At
+ first we retry at this interval but then with backoff, we pretty quickly reach
+ retrying every ten seconds. See HConstants#RETRY_BACKOFF for how the backup
+ ramps up. Change this setting and hbase.client.pause to suit your workload.</description>
+ <display-name>Maximum Client Retries</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>5</minimum>
+ <maximum>50</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.rpc.timeout</name>
+ <value>90000</value>
+ <description>
+ This is for the RPC layer to define how long HBase client applications
+ take for a remote call to time out. It uses pings to check connections
+ but will eventually throw a TimeoutException.
+ </description>
+ <display-name>HBase RPC Timeout</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>10000</minimum>
+ <maximum>180000</maximum>
+ <unit>milliseconds</unit>
+ <increment-step>10000</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.zookeeper.quorum</name>
+ <value>{{zookeeper_quorum_hosts}}</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>{{zookeeper_clientPort}}</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>zookeeper.znode.parent</name>
+ <value>{{zookeeper_znode_parent}}</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.regionserver.port</name>
+ <value>17020</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>30</value>
+ <description>
+ Count of RPC Listener instances spun up on RegionServers.
+ Same property is used by the Master for count of master handlers.
+ </description>
+ <display-name>Number of Handlers per RegionServer</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>5</minimum>
+ <maximum>240</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.hregion.majorcompaction</name>
+ <value>604800000</value>
+ <description>Time between major compactions, expressed in milliseconds. Set to 0 to disable
+ time-based automatic major compactions. User-requested and size-based major compactions will
+ still run. This value is multiplied by hbase.hregion.majorcompaction.jitter to cause
+ compaction to start at a somewhat-random time during a given window of time. The default value
+ is 7 days, expressed in milliseconds. If major compactions are causing disruption in your
+ environment, you can configure them to run at off-peak times for your deployment, or disable
+ time-based major compactions by setting this parameter to 0, and run major compactions in a
+ cron job or by another external mechanism.</description>
+ <display-name>Major Compaction Interval</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>2592000000</maximum>
+ <unit>milliseconds</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.block.multiplier</name>
+ <value>4</value>
+ <description>
+ Block updates if memstore has hbase.hregion.memstore.block.multiplier
+ times hbase.hregion.memstore.flush.size bytes. Useful preventing
+ runaway memstore during spikes in update traffic. Without an
+ upper-bound, memstore fills such that when it flushes the
+ resultant flush files take a long time to compact or split, or
+ worse, we OOME.
+ </description>
+ <display-name>HBase Region Block Multiplier</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>2</value>
+ </entry>
+ <entry>
+ <value>4</value>
+ </entry>
+ <entry>
+ <value>8</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.flush.size</name>
+ <value>134217728</value>
+ <description>
+ The size of an individual memstore. Each column familiy within each region is allocated its own memstore.
+ </description>
+ <display-name>Memstore Flush Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>33554432</minimum>
+ <maximum>268435456</maximum>
+ <increment-step>1048576</increment-step>
+ <unit>B</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.hregion.memstore.mslab.enabled</name>
+ <value>true</value>
+ <description>
+ Enables the MemStore-Local Allocation Buffer,
+ a feature which works to prevent heap fragmentation under
+ heavy write loads. This can reduce the frequency of stop-the-world
+ GC pauses on large heaps.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.hregion.max.filesize</name>
+ <value>10737418240</value>
+ <description>
+ Maximum HFile size. If the sum of the sizes of a region's HFiles has grown to exceed this
+ value, the region is split in two.
+ </description>
+ <display-name>Maximum Region File Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>1073741824</minimum>
+ <maximum>107374182400</maximum>
+ <unit>B</unit>
+ <increment-step>1073741824</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.client.scanner.caching</name>
+ <value>100</value>
+ <description>Number of rows that will be fetched when calling next
+ on a scanner if it is not served from (local, client) memory. Higher
+ caching values will enable faster scanners but will eat up more memory
+ and some calls of next may take longer and longer times when the cache is empty.
+ Do not set this value such that the time between invocations is greater
+ than the scanner timeout; i.e. hbase.regionserver.lease.period
+ </description>
+ <display-name>Number of Fetched Rows when Scanning from Disk</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>100</minimum>
+ <maximum>10000</maximum>
+ <increment-step>100</increment-step>
+ <unit>rows</unit>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.client.keyvalue.maxsize</name>
+ <value>1048576</value>
+ <description>
+ Specifies the combined maximum allowed size of a KeyValue
+ instance. This is to set an upper boundary for a single entry saved in a
+ storage file. Since they cannot be split it helps avoiding that a region
+ cannot be split any further because the data is too large. It seems wise
+ to set this to a fraction of the maximum region size. Setting it to zero
+ or less disables the check.
+ </description>
+ <display-name>Maximum Record Size</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>1048576</minimum>
+ <maximum>31457280</maximum>
+ <unit>B</unit>
+ <increment-step>262144</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.hstore.compactionThreshold</name>
+ <value>3</value>
+ <description>
+ The maximum number of StoreFiles which will be selected for a single minor
+ compaction, regardless of the number of eligible StoreFiles. Effectively, the value of
+ hbase.hstore.compaction.max controls the length of time it takes a single compaction to
+ complete. Setting it larger means that more StoreFiles are included in a compaction. For most
+ cases, the default value is appropriate.
+ </description>
+ <display-name>Maximum Store Files before Minor Compaction</display-name>
+ <value-attributes>
+ <type>int</type>
+ <entries>
+ <entry>
+ <value>2</value>
+ </entry>
+ <entry>
+ <value>3</value>
+ </entry>
+ <entry>
+ <value>4</value>
+ </entry>
+ </entries>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.hstore.blockingStoreFiles</name>
+ <display-name>hstore blocking storefiles</display-name>
+ <value>10</value>
+ <description>
+ If more than this number of StoreFiles in any one Store
+ (one StoreFile is written per flush of MemStore) then updates are
+ blocked for this HRegion until a compaction is completed, or
+ until hbase.hstore.blockingWaitTime has been exceeded.
+ </description>
+ <value-attributes>
+ <type>int</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hfile.block.cache.size</name>
+ <value>0.40</value>
+ <description>Percentage of RegionServer memory to allocate to read buffers.</description>
+ <display-name>% of RegionServer Allocated to Read Buffers</display-name>
+ <value-attributes>
+ <type>float</type>
+ <minimum>0</minimum>
+ <maximum>0.8</maximum>
+ <increment-step>0.01</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <!-- Additional configuration specific to HBase security -->
+ <property>
+ <name>hbase.superuser</name>
+ <value>yarn</value>
+ <description>List of users or groups (comma-separated), who are allowed
+ full privileges, regardless of stored ACLs, across the cluster.
+ Only used when HBase security is enabled.
+ </description>
+ <depends-on>
+ <property>
+ <type>hbase-env</type>
+ <name>hbase_user</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.security.authentication</name>
+ <value>simple</value>
+ <description>
+ Select Simple or Kerberos authentication. Note: Kerberos must be set up before the Kerberos option will take effect.
+ </description>
+ <display-name>Enable Authentication</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <label>Simple</label>
+ <value>simple</value>
+ </entry>
+ <entry>
+ <label>Kerberos</label>
+ <value>kerberos</value>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.security.authorization</name>
+ <value>false</value>
+ <description> Set Authorization Method.</description>
+ <display-name>Enable Authorization</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Native</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Off</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.coprocessor.region.classes</name>
+ <value/>
+ <description>A comma-separated list of Coprocessors that are loaded by
+ default on all tables. For any override coprocessor method, these classes
+ will be called in order. After implementing your own Coprocessor, just put
+ it in HBase's classpath and add the fully qualified class name here.
+ A coprocessor can also be loaded on demand by setting HTableDescriptor.
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.coprocessor.master.classes</name>
+ <value/>
+ <description>A comma-separated list of
+ org.apache.hadoop.hbase.coprocessor.MasterObserver coprocessors that are
+ loaded by default on the active HMaster process. For any implemented
+ coprocessor methods, the listed classes will be called in order. After
+ implementing your own MasterObserver, just put it in HBase's classpath
+ and add the fully qualified class name here.
+ </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>hbase-site</type>
+ <name>hbase.security.authorization</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.zookeeper.useMulti</name>
+ <value>true</value>
+ <description>Instructs HBase to make use of ZooKeeper's multi-update functionality.
+ This allows certain ZooKeeper operations to complete more quickly and prevents some issues
+ with rare Replication failure scenarios (see the release note of HBASE-2611 for an example).·
+ IMPORTANT: only set this to true if all ZooKeeper servers in the cluster are on version 3.4+
+ and will not be downgraded. ZooKeeper versions before 3.4 do not support multi-update and will
+ not fail gracefully if multi-update is invoked (see ZOOKEEPER-1495).
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.defaults.for.version.skip</name>
+ <value>true</value>
+ <description>Disables version verification.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>dfs.domain.socket.path</name>
+ <value>/var/lib/hadoop-hdfs/dn_socket</value>
+ <description>Path to domain socket.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.rpc.protection</name>
+ <value>authentication</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- Inherited from HBase in HDP 2.2 -->
+ <property>
+ <name>hbase.hregion.majorcompaction.jitter</name>
+ <value>0.50</value>
+ <description>A multiplier applied to hbase.hregion.majorcompaction to cause compaction to occur
+ a given amount of time either side of hbase.hregion.majorcompaction. The smaller the number,
+ the closer the compactions will happen to the hbase.hregion.majorcompaction
+ interval.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.bucketcache.ioengine</name>
+ <value/>
+ <description>Where to store the contents of the bucketcache. One of: onheap,
+ offheap, or file. If a file, set it to file:PATH_TO_FILE.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.bucketcache.size</name>
+ <value/>
+ <description>The size of the buckets for the bucketcache if you only use a single size.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.table.sanity.checks</name>
+ <value>false</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.bucketcache.percentage.in.combinedcache</name>
+ <value/>
+ <description>Value to be set between 0.0 and 1.0</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.coprocessor.regionserver.classes</name>
+ <value/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>hbase-site</type>
+ <name>hbase.security.authorization</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.hstore.compaction.max</name>
+ <value>10</value>
+ <description>The maximum number of StoreFiles which will be selected for a single minor
+ compaction, regardless of the number of eligible StoreFiles. Effectively, the value of
+ hbase.hstore.compaction.max controls the length of time it takes a single compaction to
+ complete. Setting it larger means that more StoreFiles are included in a compaction. For most
+ cases, the default value is appropriate.
+ </description>
+ <display-name>Maximum Files for Compaction</display-name>
+ <value-attributes>
+ <type>int</type>
+ <entries>
+ <entry>
+ <value>8</value>
+ </entry>
+ <entry>
+ <value>9</value>
+ </entry>
+ <entry>
+ <value>10</value>
+ </entry>
+ <entry>
+ <value>11</value>
+ </entry>
+ <entry>
+ <value>12</value>
+ </entry>
+ <entry>
+ <value>13</value>
+ </entry>
+ <entry>
+ <value>14</value>
+ </entry>
+ <entry>
+ <value>15</value>
+ </entry>
+ </entries>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.regionserver.global.memstore.size</name>
+ <value>0.4</value>
+ <description>Percentage of RegionServer memory to allocate to write buffers.
+ Each column family within each region is allocated a smaller pool (the memstore) within this shared write pool.
+ If this buffer is full, updates are blocked and data is flushed from memstores until a global low watermark
+ (hbase.regionserver.global.memstore.size.lower.limit) is reached.
+ </description>
+ <display-name>% of RegionServer Allocated to Write Buffers</display-name>
+ <value-attributes>
+ <type>float</type>
+ <minimum>0</minimum>
+ <maximum>0.8</maximum>
+ <increment-step>0.01</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- Inherited from HBase in HDP 2.5 -->
+ <property>
+ <name>hbase.master.ui.readonly</name>
+ <value>false</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>zookeeper.recovery.retry</name>
+ <value>6</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- Inherited from HBase in HDP 2.6 -->
+ <property>
+ <name>hbase.regionserver.executor.openregion.threads</name>
+ <value>20</value>
+ <description>The number of threads region server uses to open regions
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.master.namespace.init.timeout</name>
+ <value>2400000</value>
+ <description>The number of milliseconds master waits for hbase:namespace table to be initialized
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hbase.master.wait.on.regionservers.timeout</name>
+ <value>30000</value>
+ <description>The number of milliseconds master waits for region servers to report in
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+
+</configuration>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-site.xml
index c79391a67f..b1017fcaff 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/configuration/yarn-site.xml
@@ -120,7 +120,7 @@
</property>
<property>
<name>yarn.admin.acl</name>
- <value/>
+ <value>yarn,yarn-ats</value>
<description> ACL of who can be admin of the YARN cluster. </description>
<value-attributes>
<empty-value-valid>true</empty-value-valid>
@@ -183,25 +183,32 @@
<value>hadoop</value>
<description>Unix group of the NodeManager</description>
<depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
<property>
<type>cluster-env</type>
<name>user_group</name>
</property>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
- not start with numbers</description>
- <on-ambari-upgrade add="true"/>
+ <value>mapreduce_shuffle,{{timeline_collector}}</value>
+ <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and cannot start with numbers</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
<description>The auxiliary service class to use </description>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
<name>yarn.nodemanager.log-dirs</name>
@@ -491,139 +498,1145 @@
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
- <property>
- <description>Time to live for timeline store data in milliseconds.</description>
<name>yarn.timeline-service.ttl-ms</name>
+ <description>Time to live for timeline store data in milliseconds.</description>
<value>2678400000</value>
<value-attributes>
<type>int</type>
</value-attributes>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <description>Length of time to wait between deletion cycles of leveldb timeline store in milliseconds.</description>
<name>yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms</name>
+ <description>Length of time to wait between deletion cycles of leveldb timeline store in milliseconds.</description>
<value>300000</value>
<value-attributes>
<type>int</type>
</value-attributes>
<on-ambari-upgrade add="true"/>
</property>
+
+ <!-- These properties were inherited from HDP 2.2 -->
<property>
- <name>yarn.timeline-service.recovery.enabled</name>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
<description>
- Enable timeline server to recover state after starting. If
- true, then yarn.timeline-service.state-store-class must be specified.
+ List of hostname:port pairs defining the zookeeper quorum binding for the registry
</description>
- <value>true</value>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <name>yarn.acl.enable</name>
- <value>false</value>
- <description> Are acls enabled. </description>
- <on-ambari-upgrade add="true"/>
+ <name>hadoop.registry.dns.bind-port</name>
+ <value>53</value>
+ <description>
+ The port number for the DNS listener. The default port is 53.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <hidden>true</hidden>
+ <visible>false</visible>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>registry.dns.bind-port</name>
+ </property>
+ </depends-on>
</property>
<property>
- <name>yarn.authorization-provider</name>
- <description> Yarn authorization provider class. </description>
- <on-ambari-upgrade add="true"/>
+ <name>hadoop.registry.dns.zone-mask</name>
+ <value>255.255.255.0</value>
+ <description>
+ The network mask associated with the zone IP range. If specified, it is utilized to ascertain the IP range possible and come up with an appropriate reverse zone name.
+ </description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <name>yarn.admin.acl</name>
- <value>yarn</value>
- <description> ACL of who can be admin of the YARN cluster. </description>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
+ <name>hadoop.registry.dns.zone-subnet</name>
+ <value>172.17.0.0</value>
+ <description>
+ An indicator of the IP range associated with the cluster containers. The setting is utilized for the
+ generation of the reverse zone name.
+ </description>
+ <on-ambari-upgrade add="false"/>
</property>
- <!--ats v1.5 properties-->
<property>
- <name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
- <description>Main storage class for YARN timeline server.</description>
- <on-ambari-upgrade add="true"/>
+ <name>hadoop.registry.dns.enabled</name>
+ <value>true</value>
+ <description>
+ An indicator of the IP range associated with the cluster containers. The setting is utilized for the
+ generation of the reverse zone name.
+ </description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
- <value>/ats/active/</value>
- <description>DFS path to store active application’s timeline data</description>
- <on-ambari-upgrade add="true"/>
+ <name>hadoop.registry.dns.domain-name</name>
+ <value>EXAMPLE.COM</value>
+ <description>
+ The domain name for Hadoop cluster.
+ </description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
- <value>/ats/done/</value>
- <description>DFS path to store done application’s timeline data</description>
- <on-ambari-upgrade add="true"/>
+ <name>yarn.nodemanager.recovery.enabled</name>
+ <value>true</value>
+ <description>Enable the node manager to recover after starting</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
- <value/>
- <description>Plugins that can translate a timeline entity read request into a list of timeline cache ids, separated by commas. </description>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- <on-ambari-upgrade add="true"/>
+ <name>yarn.nodemanager.recovery.dir</name>
+ <display-name>YARN NodeManager Recovery directory</display-name>
+ <value>{{yarn_log_dir_prefix}}/nodemanager/recovery-state</value>
+ <description>
+ The local filesystem directory in which the node manager will store
+ state when recovery is enabled.
+ </description>
+ <on-ambari-upgrade add="false"/>
</property>
- <!-- advanced ats v1.5 properties-->
<property>
- <name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
- <description>Summary storage for ATS v1.5</description>
- <!-- Use rolling leveldb, advanced -->
- <value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value>
- <on-ambari-upgrade add="true"/>
+ <name>yarn.client.nodemanager-connect.retry-interval-ms</name>
+ <value>10000</value>
+ <description>Time interval between each attempt to connect to NM</description>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
+ <name>yarn.client.nodemanager-connect.max-wait-ms</name>
+ <value>60000</value>
+ <description>Max time to wait to establish a connection to NM</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.recovery.enabled</name>
+ <value>true</value>
<description>
- Scan interval for ATS v1.5 entity group file system storage reader.This
- value controls how frequent the reader will scan the HDFS active directory
- for application status.
+ Enable RM to recover state after starting.
+ If true, then yarn.resourcemanager.store.class must be specified.
</description>
- <!-- Default is 60 seconds, advanced -->
- <value>60</value>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
+ <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
+ <value>true</value>
<description>
- Scan interval for ATS v1.5 entity group file system storage cleaner.This
- value controls how frequent the reader will scan the HDFS done directory
- for stale application data.
+ Enable RM work preserving recovery. This configuration is private to YARN for experimenting the feature.
</description>
- <!-- 3600 is default, advanced -->
- <value>3600</value>
- <on-ambari-upgrade add="true"/>
+ <display-name>Enable Work Preserving Restart</display-name>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
</property>
<property>
- <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
+ <name>yarn.resourcemanager.store.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
<description>
- How long the ATS v1.5 entity group file system storage will keep an
- application's data in the done directory.
+ The class to use as the persistent store.
+ If org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore is used,
+ the store is implicitly fenced; meaning a single ResourceManager
+ is able to use the store at any point in time.
</description>
- <!-- 7 days is default, advanced -->
- <value>604800</value>
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>yarn.log.server.web-service.url</name>
- <value>http://localhost:8188/ws/v1/applicationhistory</value>
- <description>Log Server Web Service URL.</description>
+ <name>yarn.resourcemanager.zk-address</name>
+ <value>localhost:2181</value>
+ <description>
+ List Host:Port of the ZooKeeper servers to be used by the RM. comma separated host:port pairs, each corresponding to a zk server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" If the optional chroot suffix is used the example would look like: "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002/app/a" where the client would be rooted at "/app/a" and all paths would be relative to this root - ie getting/setting/etc... "/foo/bar" would result in operations being run on "/app/a/foo/ba [...]
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-state-store.parent-path</name>
+ <value>/rmstore</value>
+ <description>Full path of the ZooKeeper znode where RM state will be stored. This must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore as the value for yarn.resourcemanager.store.class</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-acl</name>
+ <value>world:anyone:rwcda</value>
+ <description>ACL's to be used for ZooKeeper znodes.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms</name>
+ <value>10000</value>
+ <description>Set the amount of time RM waits before allocating new containers on work-preserving-recovery. Such wait period gives RM a chance to settle down resyncing with NMs in the cluster on recovery, before assigning new containers to applications.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.connect.retry-interval.ms</name>
+ <value>30000</value>
+ <description>How often to try connecting to the ResourceManager.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.connect.max-wait.ms</name>
+ <value>900000</value>
+ <description>Maximum time to wait to establish connection to ResourceManager</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-retry-interval-ms</name>
+ <value>1000</value>
+ <description>"Retry interval in milliseconds when connecting to ZooKeeper.
+ When HA is enabled, the value here is NOT used. It is generated
+ automatically from yarn.resourcemanager.zk-timeout-ms and
+ yarn.resourcemanager.zk-num-retries."
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-num-retries</name>
+ <value>1000</value>
+ <description>Number of times RM tries to connect to ZooKeeper.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.zk-timeout-ms</name>
+ <value>10000</value>
+ <description>ZooKeeper session timeout in milliseconds. Session expiration is managed by the ZooKeeper cluster itself, not by the client. This value is used by the cluster to determine when the client's session expires. Expirations happens when the cluster does not hear from the client within the specified session timeout period (i.e. no heartbeat).</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.state-store.max-completed-applications</name>
+ <value>${yarn.resourcemanager.max-completed-applications}</value>
+ <description>The maximum number of completed applications RM state store keeps, less than or equals to ${yarn.resourcemanager.max-completed-applications}. By default, it equals to ${yarn.resourcemanager.max-completed-applications}. This ensures that the applications kept in the state store are consistent with the applications remembered in RM memory. Any values larger than ${yarn.resourcemanager.max-completed-applications} will be reset to ${yarn.resourcemanager.max-completed-applica [...]
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.fs.state-store.retry-policy-spec</name>
+ <value>2000, 500</value>
+ <description>hdfs client retry policy specification. hdfs client retry is always enabled. Specified in pairs of sleep-time and number-of-retries and (t0, n0), (t1, n1), ..., the first n0 retries sleep t0 milliseconds on average, the following n1 retries sleep t1 milliseconds on average, and so on.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.fs.state-store.uri</name>
+ <value> </value>
+ <description>RI pointing to the location of the FileSystem path where RM state will be stored. This must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore as the value for yarn.resourcemanager.store.class </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.ha.enabled</name>
+ <value>false</value>
+ <description>enable RM HA or not</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.resources-handler.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler</value>
+ <description>Pre-requisite to use CGroups</description>
<depends-on>
<property>
- <type>yarn-site</type>
- <name>yarn.http.policy</name>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
</property>
<property>
- <type>yarn-site</type>
- <name>yarn.timeline-service.webapp.address</name>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.hierarchy</name>
+ <value>/yarn</value>
+ <description>Name of the Cgroups hierarchy under which all YARN jobs will be launched</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.mount</name>
+ <value>false</value>
+ <description>If true, YARN will automount the CGroup, however the directory needs to already exist; else, the cgroup should be mounted by the admin</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.mount-path</name>
+ <value>/cgroup</value>
+ <description>Path used by the LCE to mount cgroups if not found. This path must exist before the NodeManager is launched.</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
</property>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage</name>
+ <value>false</value>
+ <description>Strictly limit CPU resource usage to allocated usage even if spare CPU is available</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>8</value>
+ <description>Number of vcores that can be allocated
+ for containers. This is used by the RM scheduler when allocating
+ resources for containers. This is not used to limit the number of
+ CPUs used by YARN containers. If it is set to -1 and
+ yarn.nodemanager.resource.detect-hardware-capabilities is true, it is
+ automatically determined from the hardware in case of Windows and Linux.
+ In other cases, number of vcores is 8 by default.
+ </description>
+ <display-name>Number of virtual cores</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>32</maximum>
+ </value-attributes>
+ <depends-on>
<property>
<type>yarn-site</type>
- <name>yarn.timeline-service.webapp.https.address</name>
+ <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
</property>
</depends-on>
- <on-ambari-upgrade add="true"/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource.percentage-physical-cpu-limit</name>
+ <value>80</value>
+ <description>The amount of CPU allocated for YARN containers - only effective when used with CGroups</description>
+ <display-name>Percentage of physical CPU allocated for all containers on a node</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>100</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.node-labels.fs-store.retry-policy-spec</name>
+ <value>2000, 500</value>
+ <description>
+ Retry policy used for FileSystem node label store. The policy is
+ specified by N pairs of sleep-time in milliseconds and number-of-retries
+ "s1,n1,s2,n2,...".
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb</name>
+ <value>1000</value>
+ <description>This is related to disk size on the machines, admins should set one of yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb or yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage but not both. If both are set, the more conservative value will be used</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage</name>
+ <value>90</value>
+ <description>This is related to disk size on the machines, admins should set one of yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb or yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage but not both. If both are set, the more conservative value will be used</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource-plugins</name>
+ <value></value>
+ <description>
+ Enable additional discovery/isolation of resources on the NodeManager,
+ split by comma. By default, this is empty.
+ Acceptable values: { "yarn-io/gpu"}
+ </description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices</name>
+ <value>auto</value>
+ <display-name>GPU allowed devices</display-name>
+ <description>Specify GPU devices which can be managed by YARN NodeManager,
+ split by comma. Number of GPU devices will be reported to RM to make
+ scheduling decisions. Set to auto (default) let YARN automatically
+ discover GPU resource from system
+ </description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource-plugins.gpu.path-to-discovery-executables</name>
+ <value></value>
+ <display-name>Absolute path of nvidia-smi on NodeManagers</display-name>
+ <description>When value is empty (default), YARN NodeManager will try to
+ locate discovery executable itself</description>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <type>directories</type>
+ <overridable>false</overridable>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+ <description>Defines how often NMs wake up to upload log files. The default value is -1. By default, the logs will be uploaded whenthe application is finished. By setting this configure, logs can be uploaded periodically when the application is running. The minimum rolling-interval-seconds can be set is 3600.</description>
+ <value>3600</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.debug-enabled</name>
+ <value>false</value>
+ <description>
+ This configuration is for debug and test purpose.
+ By setting this configuration as true.
+ We can break the lower bound of yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
+ <value>30</value>
+ <description>This is temporary solution. The configuration will be deleted once, we find a more scalable method to only write a single log file per LRS.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+ <value>true</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
+ <value>10</value>
+ <description>Number of worker threads that send the yarn system metrics data.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.client.max-retries</name>
+ <value>30</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.client.retry-interval-ms</name>
+ <value>1000</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.ttl-enable</name>
+ <value>true</value>
+ <description>
+ Enable age off of timeline store data.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.state-store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore</value>
+ <description>Store class name for timeline state store.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-state-store.path</name>
+ <value>/hadoop/yarn/timeline</value>
+ <description>Store file name for leveldb state store.</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.path</name>
+ <value>/hadoop/yarn/timeline</value>
+ <description>Store file name for leveldb timeline store.</description>
+ <value-attributes>
+ <type>directory</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.read-cache-size</name>
+ <value>104857600</value>
+ <description>
+ Size of read cache for uncompressed blocks for leveldb timeline store in bytes.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size</name>
+ <value>10000</value>
+ <description>
+ Size of cache for recently read entity start times for leveldb timeline store in number of entities.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size</name>
+ <value>10000</value>
+ <description>
+ Size of cache for recently written entity start times for leveldb timeline store in number of entities.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.http-authentication.type</name>
+ <value>simple</value>
+ <description>
+ Defines authentication used for the Timeline Server HTTP endpoint.
+ Supported values are: simple | kerberos | $AUTHENTICATION_HANDLER_CLASSNAME
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>false</value>
+ <description>
+ Flag to enable override of the default kerberos authentication filter with
+ the RM authentication filter to allow authentication using delegation
+ tokens(fallback to kerberos if the tokens are missing).
+ Only applicable when the http authentication type is kerberos.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.bind-host</name>
+ <value>0.0.0.0</value>
+ <description>Default value is 0.0.0.0, when this is set the service will bind on all interfaces. If this optional address is set, the RPC and webapp servers will bind to this address and the port specified in yarn.resourcemanager.address and yarn.resourcemanager.webapp.address, respectively.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.bind-host</name>
+ <value>0.0.0.0</value>
+ <description>Default value is 0.0.0.0, when this is set the service will bind on all interfaces. If this optional address is set, the RPC and webapp servers will bind to this address and the port specified in yarn.nodemanager.address and yarn.nodemanager.webapp.address, respectively.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.bind-host</name>
+ <value>0.0.0.0</value>
+ <description>Default value is 0.0.0.0, when this is set the service will bind on all interfaces. If this optional address is set, the RPC and webapp servers will bind to this address and the port specified in yarn.timeline-service.address and yarn.timeline-service.webapp.address, respectively.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.node-labels.fs-store.root-dir</name>
+ <display-name>YARN Node Labels FS Store Root directory</display-name>
+ <value>/system/yarn/node-labels</value>
+ <description>
+ URI for NodeLabelManager.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>1</value>
+ <description/>
+ <display-name>Minimum Container Size (VCores)</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>8</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.scheduler.maximum-allocation-vcores</name>
+ <value>8</value>
+ <description/>
+ <display-name>Maximum Container Size (VCores)</display-name>
+ <value-attributes>
+ <type>int</type>
+ <minimum>0</minimum>
+ <maximum>8</maximum>
+ <increment-step>1</increment-step>
+ </value-attributes>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.node-labels.enabled</name>
+ <value>false</value>
+ <description>
+ Enable node labels to restrict YARN applications so that they run only on cluster nodes that have a specified node label.
+ </description>
+ <display-name>Node Labels</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
+ <description>ContainerExecutor for launching containers</description>
+ <depends-on>
+ <property>
+ <type>yarn-env</type>
+ <name>yarn_cgroups_enabled</name>
+ </property>
+ <property>
+ <type>core-site</type>
+ <name>hadoop.security.authentication</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.scheduler.monitor.enable</name>
+ <description>
+ Enable a set of periodic monitors (specified in
+ yarn.resourcemanager.scheduler.monitor.policies) that affect the
+ scheduler.
+ </description>
+ <value>true</value>
+ <display-name>Pre-emption</display-name>
+ <value-attributes>
+ <type>value-list</type>
+ <entries>
+ <entry>
+ <value>true</value>
+ <label>Enabled</label>
+ </entry>
+ <entry>
+ <value>false</value>
+ <label>Disabled</label>
+ </entry>
+ </entries>
+ <selection-cardinality>1</selection-cardinality>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <!-- In HDP 2.3, these properties were deleted:
+yarn.node-labels.manager-class
+-->
+
+ <!-- These configs were inherited from HDP 2.3 -->
+ <property>
+ <name>yarn.timeline-service.recovery.enabled</name>
+ <description>
+ Enable timeline server to recover state after starting. If
+ true, then yarn.timeline-service.state-store-class must be specified.
+ </description>
+ <value>true</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.authorization-provider</name>
+ <description> Yarn authorization provider class. </description>
+ <depends-on>
+ <property>
+ <type>ranger-yarn-plugin-properties</type>
+ <name>ranger-yarn-plugin-enabled</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!--ats v2.0 properties-->
+ <property>
+ <name>yarn.timeline-service.version</name>
+ <value>2.0f</value>
+ <description>Timeline service version we’re currently using.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.versions</name>
+ <value>1.5f,2.0f</value>
+ <description>Comma separated Timeline service versions we’re currently using. This overrides yarn.timeline-service.version. Acceptable values are 1.5f or 2.0f</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.system-metricspublisher.enabled</name>
+ <value>true</value>
+ <description>Enables YARN to publish metrics to timeline v2</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.rm.system-metricspublisher.emit-container-events</name>
+ <value>true</value>
+ <description>Experimentally enable each container to post timeline events to timeline v2</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.recovery.supervised</name>
+ <value>true</value>
+ <description>Default of false causes NM to kill its containers on shutdown. Should be set to true when NM recovery is enabled.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.store-class</name>
+ <value>org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore</value>
+ <description>Main storage class for YARN timeline server.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.active-dir</name>
+ <display-name>YARN Timeline Service Entity Group FS Store Active directory</display-name>
+ <value>/ats/active/</value>
+ <description>DFS path to store active application’s timeline data</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.done-dir</name>
+ <display-name>YARN Timeline Service Entity Group FS Store Done directory</display-name>
+ <value>/ats/done/</value>
+ <description>DFS path to store done application’s timeline data</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes</name>
+ <value>org.apache.hadoop.yarn.applications.distributedshell.DistributedShellTimelinePlugin</value>
+ <description>Plugins that can translate a timeline entity read request into a list of timeline cache ids, separated by commas. </description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <!-- advanced ats v1.5 properties-->
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.summary-store</name>
+ <description>Summary storage for ATS v1.5</description>
+ <!-- Use rolling leveldb, advanced -->
+ <value>org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.scan-interval-seconds</name>
+ <description>
+ Scan interval for ATS v1.5 entity group file system storage reader.This
+ value controls how frequent the reader will scan the HDFS active directory
+ for application status.
+ </description>
+ <!-- Default is 60 seconds, advanced -->
+ <value>60</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.log.server.web-service.url</name>
+ <value>http://localhost:8188/ws/v1/applicationhistory</value>
+ <description>Log Server Web Service URL.</description>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.http.policy</name>
+ </property>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.timeline-service.webapp.address</name>
+ </property>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.timeline-service.webapp.https.address</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds</name>
+ <description>
+ Scan interval for ATS v1.5 entity group file system storage cleaner.This
+ value controls how frequent the reader will scan the HDFS done directory
+ for stale application data.
+ </description>
+ <!-- 3600 is default, advanced -->
+ <value>3600</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.retain-seconds</name>
+ <description>
+ How long the ATS v1.5 entity group file system storage will keep an
+ application's data in the done directory.
+ </description>
+ <!-- 7 days is default, advanced -->
+ <value>604800</value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+
+ <!-- These configs were inherited from HDP 2.5 -->
+ <property>
+ <name>yarn.nodemanager.container-metrics.unregister-delay-ms</name>
+ <value>60000</value>
+ <description>The delay time ms to unregister container metrics after completion.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath</name>
+ <value/>
+ <description>Classpath for all plugins defined in yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes.</description>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round</name>
+ <value>0.1</value>
+ <description>This option controls the pace at which containers-marked-for-preemption are actually preempted in each period.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor</name>
+ <value>1</value>
+ <description>Similar to total_preemption_per_round, we can apply this factor to slowdown resource preemption after preemption-target is computed for each queue.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.monitoring_interval</name>
+ <value>15000</value>
+ <description>Time in milliseconds between invocations of this ProportionalCapacityPreemptionPolicy policy.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!-- These configs are related to docker -->
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users</name>
+ <value>true</value>
+ <description>
+ Whether all applications should be run as the NodeManager process' owner.
+ When false, applications are launched instead as the application owner.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.runtime.linux.allowed-runtimes</name>
+ <value>default,docker</value>
+ <description>
+ Comma separated list of runtimes that are allowed when using
+ LinuxContainerExecutor. The allowed values are default, docker, and
+ javasandbox.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.allowed-container-networks</name>
+ <value>host,none,bridge</value>
+ <description>
+ Optional. A comma-separated set of networks allowed when launching
+ containers. Valid values are determined by Docker networks available from
+ `docker network ls`.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.default-container-network</name>
+ <value>host</value>
+ <description>
+ The network used when launching Docker containers when no
+ network is specified in the request. This network must be one of the
+ (configurable) set of allowed container networks.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed</name>
+ <value>false</value>
+ <description>
+ Optional. Whether applications are allowed to run in privileged
+ containers.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.privileged-containers.acl</name>
+ <value/>
+ <description>
+ Optional. A comma-separated list of users who are allowed to request
+ privileged contains if privileged containers are allowed.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ </property>
+ <property>
+ <name>yarn.nodemanager.runtime.linux.docker.capabilities</name>
+ <value>
+ CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,
+ SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE
+ </value>
+ <description>
+ Optional. This configuration setting determines the capabilities
+ assigned to docker containers when they are launched. While these may not
+ be case-sensitive from a docker perspective, it is best to keep these
+ uppercase. To run without any capabilites, set this value to
+ "none" or "NONE"
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.webapp.ui2.enable</name>
+ <value>true</value>
+ <description>
+ This configuration setting enables YARN web ui2 server which is a revamped
+ UI for better user experience.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!--Enable CORS for all component-->
+ <property>
+ <name>yarn.timeline-service.http-cross-origin.enabled</name>
+ <value>true</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.webapp.cross-origin.enabled</name>
+ <value>true</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.webapp.cross-origin.enabled</name>
+ <value>true</value>
+ <description/>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource-plugins.gpu.docker-plugin</name>
+ <value>nvidia-docker-v1</value>
+ <display-name>GPU docker plugin</display-name>
+ <description>Specify docker command plugin for GPU. By default uses Nvidia
+ docker V1.0
+ </description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>docker_module_enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidiadocker-v1.endpoint</name>
+ <value>http://localhost:3476/v1.0/docker/cli</value>
+ <display-name>GPU docker plugin endpoint for Nvidia Docker Version 1</display-name>
+ <description>Specify end point of nvidia-docker-plugin</description>
+ <depends-on>
+ <property>
+ <type>container-executor</type>
+ <name>gpu_module_enabled</name>
+ </property>
+ <property>
+ <type>container-executor</type>
+ <name>docker_module_enabled</name>
+ </property>
+ </depends-on>
+ <value-attributes>
+ <empty-value-valid>true</empty-value-valid>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <!-- These configs are related to yarn service -->
+ <property>
+ <name>yarn.webapp.api-service.enable</name>
+ <value>true</value>
+ <description>
+ Whether to enable Yarn API Service.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.service.framework.path</name>
+ <value>/bigtop/apps/3.2.0/yarn/service-dep.tar.gz</value>
+ <description>
+ YARN Service AM dependency tarball location. Enables faster app submission.
+ </description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+
+ <!--ATSv2 properties-->
+ <property>
+ <name>yarn.nodemanager.aux-services.timeline_collector.class</name>
+ <value>org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService</value>
+ <description>The auxiliary service class to use </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.reader.webapp.address</name>
+ <value>localhost:8198</value>
+ <description>
+ The http address of the timeline reader web application.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.reader.webapp.https.address</name>
+ <value>localhost:8199</value>
+ <description>
+ The http address of the timeline reader web application.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+
+ <!--ATSv2 HBase Properties-->
+ <property>
+ <name>yarn.timeline-service.hbase-schema.prefix</name>
+ <value>prod.</value>
+ <description>HBase table prefix.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.hbase.configuration.file</name>
+ <value>file://{{yarn_hbase_conf_dir}}/hbase-site.xml</value>
+ <description>HBase configuration location.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.hbase.coprocessor.jar.hdfs.location</name>
+ <value>{{yarn_timeline_jar_location}}</value>
+ <description>HBase table creation jar location.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.monitor.capacity.preemption.intra-queue-preemption.enabled</name>
+ <value>true</value>
+ <description>This property need to be set to make preemption within a Leaf Queue based on user-limit and application priority.</description>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.resourcemanager.scheduler.monitor.enable</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled</name>
+ <value>true</value>
+ <description>This property need to be set to make preemption can happen under a fragmented cluster.</description>
+ <depends-on>
+ <property>
+ <type>yarn-site</type>
+ <name>yarn.resourcemanager.scheduler.monitor.enable</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.display.per-user-apps</name>
+ <value>true</value>
+ <description>
+ Flag to enable display of applications per user as an admin
+ configuration.
+ </description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.service.system-service.dir</name>
+ <value>/services</value>
+ <description>FS directory path to load and deploy admin configured services</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.timeline-service.generic-application-history.save-non-am-container-meta-info</name>
+ <value>false</value>
+ <description>Defines if RM will save non am container meta-info in history store.</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hadoop.registry.dns.bind-address</name>
+ <value>0.0.0.0</value>
+ <description>Address associated with the network interface to which the DNS listener should bind.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>hadoop.http.cross-origin.allowed-origins</name>
+ <value>{{cross_origins}}</value>
+ <description>Comma separated list of origins that are allowed for web services
+ needing cross-origin (CORS) support.</description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>yarn.nodemanager.resourcemanager.connect.wait.secs</name>
+ <value>1800</value>
+ <description>Max time, in seconds, to wait to establish a connection to RM when NM starts. The NM will shutdown if it cannot connect to RM within the specified max time period. If the value is set as -1, then NM will retry forever.</description>
+ <on-ambari-upgrade add="false"/>
</property>
</configuration>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/kerberos.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/kerberos.json
index e5ce2b861a..a34c458a16 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/kerberos.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/kerberos.json
@@ -10,14 +10,78 @@
{
"name": "yarn_smokeuser",
"reference": "/smokeuser"
+ },
+ {
+ "name": "yarn_ats",
+ "principal": {
+ "value": "${yarn-env/yarn_ats_user}${principal_suffix}@${realm}",
+ "type" : "user",
+ "configuration": "yarn-env/yarn_ats_principal_name",
+ "local_username": "${yarn-env/yarn_ats_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/yarn-ats.hbase-client.headless.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_ats_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-env/yarn_ats_user_keytab"
+ }
+ },
+ {
+ "name": "yarn_ats_hbase_master",
+ "principal": {
+ "value": "yarn-ats-hbase/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-hbase-site/hbase.master.kerberos.principal",
+ "local_username": "${yarn-env/yarn_ats_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/yarn-ats.hbase-master.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_ats_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-hbase-site/hbase.master.keytab.file"
+ }
+ },
+ {
+ "name": "yarn_ats_hbase_regionserver",
+ "principal": {
+ "value": "yarn-ats-hbase/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-hbase-site/hbase.regionserver.kerberos.principal",
+ "local_username": "${yarn-env/yarn_ats_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/yarn-ats.hbase-regionserver.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_ats_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-hbase-site/hbase.regionserver.keytab.file"
+ }
}
],
"configurations": [
{
"yarn-site": {
- "yarn.timeline-service.enabled": "false",
+ "yarn.timeline-service.enabled": "true",
"yarn.timeline-service.http-authentication.type": "kerberos",
"yarn.acl.enable": "true",
+ "yarn.admin.acl": "${yarn-env/yarn_user}",
"yarn.timeline-service.http-authentication.signature.secret": "",
"yarn.timeline-service.http-authentication.signature.secret.file": "",
"yarn.timeline-service.http-authentication.signer.secret.provider": "",
@@ -33,8 +97,11 @@
"yarn.resourcemanager.proxyuser.*.hosts": "",
"yarn.resourcemanager.proxyuser.*.users": "",
"yarn.resourcemanager.proxy-user-privileges.enabled": "true",
+ "yarn.resourcemanager.zk-acl" : "sasl:${principals/YARN/RESOURCEMANAGER/resource_manager_rm|principalPrimary()}:rwcda",
"hadoop.registry.secure" : "true",
- "hadoop.registry.system.accounts" : "sasl:yarn,sasl:mapred,sasl:hadoop,sasl:hdfs,sasl:rm"
+ "hadoop.registry.system.accounts" : "sasl:${principals/YARN/APP_TIMELINE_SERVER/app_timeline_server_yarn|principalPrimary()},sasl:${principals/MAPREDUCE2/HISTORYSERVER/history_server_jhs|principalPrimary()},sasl:${principals/HDFS/NAMENODE/hdfs|principalPrimary()},sasl:${principals/YARN/RESOURCEMANAGER/resource_manager_rm|principalPrimary()},sasl:${principals/HIVE/HIVE_SERVER/hive_server_hive|principalPrimary()},sasl:${principals/SPARK2/spark_service_keytab|principalPrimary()}",
+ "hadoop.registry.client.auth" : "kerberos",
+ "hadoop.registry.jaas.context" : "Client"
}
},
{
@@ -42,6 +109,34 @@
"hadoop.proxyuser.${yarn-env/yarn_user}.groups": "*",
"hadoop.proxyuser.${yarn-env/yarn_user}.hosts": "${clusterHostInfo/resourcemanager_hosts}"
}
+ },
+ {
+ "capacity-scheduler": {
+ "yarn.scheduler.capacity.root.acl_submit_applications": "${yarn-env/yarn_user},${cluster-env/smokeuser}",
+ "yarn.scheduler.capacity.root.acl_administer_queue": "${yarn-env/yarn_user}",
+ "yarn.scheduler.capacity.root.default.acl_administer_queue": "${yarn-env/yarn_user}",
+ "yarn.scheduler.capacity.root.default.acl_administer_jobs": "${yarn-env/yarn_user}",
+ "yarn.scheduler.capacity.root.default.acl_submit_applications": "${yarn-env/yarn_user},${yarn-env/yarn_ats_user}"
+ }
+ },
+ {
+ "ranger-yarn-audit": {
+ "xasecure.audit.jaas.Client.loginModuleName": "com.sun.security.auth.module.Krb5LoginModule",
+ "xasecure.audit.jaas.Client.loginModuleControlFlag": "required",
+ "xasecure.audit.jaas.Client.option.useKeyTab": "true",
+ "xasecure.audit.jaas.Client.option.storeKey": "false",
+ "xasecure.audit.jaas.Client.option.serviceName": "solr",
+ "xasecure.audit.destination.solr.force.use.inmemory.jaas.config": "true"
+ }
+ },
+ {
+ "yarn-hbase-site": {
+ "hbase.security.authentication": "kerberos",
+ "hbase.security.authorization": "true",
+ "hbase.coprocessor.master.classes": "org.apache.hadoop.hbase.security.access.AccessController",
+ "hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.AccessController",
+ "zookeeper.znode.parent": "/atsv2-hbase-secure"
+ }
}
],
"components": [
@@ -69,6 +164,42 @@
"configuration": "yarn-site/yarn.nodemanager.keytab"
}
},
+ {
+ "name": "yarn_nodemanager_hive_server_hive",
+ "reference": "/HIVE/HIVE_SERVER/hive_server_hive",
+ "principal": {
+ "configuration": "hive-interactive-site/hive.llap.daemon.service.principal"
+ },
+ "keytab": {
+ "configuration": "hive-interactive-site/hive.llap.daemon.keytab.file"
+ },
+ "when" : {
+ "contains" : ["services", "HIVE"]
+ }
+ },
+ {
+ "name": "llap_task_hive",
+ "principal": {
+ "value": "hive/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "hive-interactive-site/hive.llap.task.principal"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/hive.llap.task.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": "r"
+ },
+ "configuration": "hive-interactive-site/hive.llap.task.keytab.file"
+ },
+ "when" : {
+ "contains" : ["services", "HIVE"]
+ }
+ },
{
"name": "yarn_nodemanager_spnego",
"reference": "/spnego",
@@ -91,6 +222,10 @@
{
"name": "RESOURCEMANAGER",
"identities": [
+ {
+ "name": "yarn_resourcemanager_hdfs",
+ "reference": "/HDFS/NAMENODE/hdfs"
+ },
{
"name": "resource_manager_rm",
"principal": {
@@ -121,6 +256,16 @@
"keytab": {
"configuration": "yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file"
}
+ },
+ {
+ "name": "yarn_resourcemanager_resource_manager_rm",
+ "reference": "/YARN/RESOURCEMANAGER/resource_manager_rm",
+ "principal": {
+ "configuration": "ranger-yarn-audit/xasecure.audit.jaas.Client.option.principal"
+ },
+ "keytab": {
+ "configuration": "ranger-yarn-audit/xasecure.audit.jaas.Client.option.keyTab"
+ }
}
]
},
@@ -163,6 +308,72 @@
"reference": "/HDFS/NAMENODE/hdfs"
}
]
+ },
+ {
+ "name": "TIMELINE_READER",
+ "identities": [
+ {
+ "name": "yarn_timeline_reader",
+ "principal": {
+ "value": "yarn/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-site/yarn.timeline-service.principal",
+ "local_username": "${yarn-env/yarn_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/yarn.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-site/yarn.timeline-service.keytab"
+ }
+ },
+ {
+ "name": "yarn_timeline_reader_spnego",
+ "reference": "/spnego",
+ "principal": {
+ "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.principal"
+ },
+ "keytab": {
+ "configuration": "yarn-site/yarn.timeline-service.http-authentication.kerberos.keytab"
+ }
+ },
+ {
+ "name": "yarn_timeline_reader_hdfs",
+ "reference": "/HDFS/NAMENODE/hdfs"
+ }
+ ]
+ },
+ {
+ "name": "YARN_REGISTRY_DNS",
+ "identities": [
+ {
+ "name": "registry_dns_yarn",
+ "principal": {
+ "value": "yarn/_HOST@${realm}",
+ "type" : "service",
+ "configuration": "yarn-env/yarn.registry-dns.principal",
+ "local_username": "${yarn-env/yarn_user}"
+ },
+ "keytab": {
+ "file": "${keytab_dir}/yarn.service.keytab",
+ "owner": {
+ "name": "${yarn-env/yarn_user}",
+ "access": "r"
+ },
+ "group": {
+ "name": "${cluster-env/user_group}",
+ "access": ""
+ },
+ "configuration": "yarn-env/yarn.registry-dns.keytab"
+ }
+ }
+ ]
}
]
},
@@ -178,6 +389,14 @@
"reference": "/smokeuser"
}
],
+ "configurations": [
+ {
+ "mapred-site": {
+ "mapreduce.cluster.acls.enabled": "true",
+ "mapreduce.jobhistory.admin.acl": "${mapred-env/mapred_user}"
+ }
+ }
+ ],
"components": [
{
"name": "HISTORYSERVER",
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/metainfo.xml
index 64b41343d2..cbfadc2028 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/metainfo.xml
@@ -26,6 +26,72 @@
<version>3.3.4-1</version>
<components>
+ <component>
+ <name>TIMELINE_READER</name>
+ <displayName>Timeline Service V2.0 Reader</displayName>
+ <category>MASTER</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>true</reassignAllowed>
+ <auto-deploy>
+ <enabled>true</enabled>
+ <co-locate>YARN/RESOURCEMANAGER</co-locate>
+ </auto-deploy>
+
+ <commandScript>
+ <script>scripts/timelinereader.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <!-- ATSv2 backend i.e Hbase requires HDFS as dependency -->
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ </component>
+
+ <component>
+ <name>YARN_REGISTRY_DNS</name>
+ <displayName>YARN Registry DNS</displayName>
+ <category>MASTER</category>
+ <cardinality>0-1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>true</reassignAllowed>
+
+ <commandScript>
+ <script>scripts/yarn_registry_dns.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>APP_TIMELINE_SERVER</name>
+ <displayName>Timeline Service V2</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <reassignAllowed>true</reassignAllowed>
+
+ <commandScript>
+ <script>scripts/application_timeline_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+
+ <logs>
+ <log>
+ <logId>yarn_timelineserver</logId>
+ </log>
+ </logs>
+
+ </component>
+
<component>
<name>RESOURCEMANAGER</name>
<displayName>ResourceManager</displayName>
@@ -72,6 +138,8 @@
<configuration-dependencies>
<config-type>capacity-scheduler</config-type>
<config-type>hdfs-site</config-type>
+ <config-type>resource-types</config-type>
+ <config-type>container-executor</config-type>
</configuration-dependencies>
</component>
@@ -98,6 +166,11 @@
<logId>yarn_nodemanager</logId>
</log>
</logs>
+
+ <configuration-dependencies>
+ <config-type>resource-types</config-type>
+ <config-type>container-executor</config-type>
+ </configuration-dependencies>
</component>
<component>
@@ -137,6 +210,16 @@
<fileName>capacity-scheduler.xml</fileName>
<dictionaryName>capacity-scheduler</dictionaryName>
</configFile>
+ <configFile>
+ <type>xml</type>
+ <fileName>resource-types.xml</fileName>
+ <dictionaryName>resource-types</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>properties</type>
+ <fileName>container-executor.cfg</fileName>
+ <dictionaryName>container-executor</dictionaryName>
+ </configFile>
</configFiles>
</component>
</components>
@@ -186,13 +269,16 @@
<config-type>hdfs-site</config-type>
<config-type>hadoop-env</config-type>
<config-type>core-site</config-type>
+ <config-type>viewfs-mount-table</config-type>
<config-type>mapred-site</config-type>
<config-type>yarn-log4j</config-type>
+ <config-type>ams-ssl-client</config-type>
<config-type>ranger-yarn-plugin-properties</config-type>
<config-type>ranger-yarn-audit</config-type>
<config-type>ranger-yarn-policymgr-ssl</config-type>
<config-type>ranger-yarn-security</config-type>
</configuration-dependencies>
+
<widgetsFileName>YARN_widgets.json</widgetsFileName>
<metricsFileName>YARN_metrics.json</metricsFileName>
<themes>
@@ -326,10 +412,12 @@
<config-type>hdfs-site</config-type>
<config-type>hadoop-env</config-type>
<config-type>core-site</config-type>
+ <config-type>viewfs-mount-table</config-type>
<config-type>mapred-site</config-type>
<config-type>mapred-env</config-type>
<config-type>ssl-client</config-type>
<config-type>ssl-server</config-type>
+ <config-type>ams-ssl-client</config-type>
<config-type>yarn-site</config-type>
</configuration-dependencies>
<restartRequiredAfterRackChange>true</restartRequiredAfterRackChange>
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/alerts/alert_ats_hbase.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/alerts/alert_ats_hbase.py
new file mode 100755
index 0000000000..118062a5cf
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/alerts/alert_ats_hbase.py
@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import logging
+import json
+import subprocess
+import time
+import traceback
+
+from resource_management.core import global_lock
+from resource_management.core import shell
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources import Execute
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.core.exceptions import ComponentIsNotRunning
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+CRITICAL_RESULT_CODE = 'CRITICAL'
+OK_RESULT_CODE = 'OK'
+UKNOWN_STATUS_CODE = 'UNKNOWN'
+
+OK_MESSAGE = "The HBase application reported a '{0}' state in {1:.3f}s"
+MESSAGE_WITH_STATE_AND_INSTANCES = "The application reported a '{0}' state in {1:.3f}s. [Live: {2}, Desired: {3}]"
+CRITICAL_MESSAGE_WITH_STATE = "The HBase application reported a '{0}' state. Check took {1:.3f}s"
+CRITICAL_MESSAGE = "ats-hbase service information could not be retrieved"
+
+
+SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
+STACK_ROOT = '{{cluster-env/stack_root}}'
+STACK_ROOT_DEFAULT = Script.get_stack_root()
+
+
+ATS_HBASE_PRINCIPAL_KEY = '{{yarn-hbase-site/hbase.master.kerberos.principal}}'
+ATS_HBASE_PRINCIPAL_KEYTAB_KEY = '{{yarn-hbase-site/hbase.master.keytab.file}}'
+ATS_HBASE_USER_KEY = '{{yarn-env/yarn_ats_user}}'
+ATS_HBASE_SYSTEM_SERVICE_LAUNCH_KEY = '{{yarn-hbase-env/is_hbase_system_service_launch}}'
+USE_EXTERNAL_HBASE_KEY = '{{yarn-hbase-env/use_external_hbase}}'
+ATS_HBASE_PID_DIR_PREFIX = '{{yarn-hbase-env/yarn_hbase_pid_dir_prefix}}'
+
+ATS_HBASE_APP_NOT_FOUND_KEY = format("Service ats-hbase not found")
+
+# The configured Kerberos executable search paths, if any
+KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}}'
+
+
+CHECK_COMMAND_TIMEOUT_KEY = 'check.command.timeout'
+CHECK_COMMAND_TIMEOUT_DEFAULT = 120.0
+
+
+logger = logging.getLogger('ambari_alerts')
+
+
+def get_tokens():
+ """
+ Returns a tuple of tokens in the format {{site/property}} that will be used
+ to build the dictionary passed into execute
+ """
+ return (SECURITY_ENABLED_KEY, KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY, ATS_HBASE_PRINCIPAL_KEY, ATS_HBASE_PRINCIPAL_KEYTAB_KEY,
+ ATS_HBASE_USER_KEY, STACK_ROOT, USE_EXTERNAL_HBASE_KEY, ATS_HBASE_PID_DIR_PREFIX, ATS_HBASE_SYSTEM_SERVICE_LAUNCH_KEY)
+
+
+def execute(configurations={}, parameters={}, host_name=None):
+ """
+ Returns a tuple containing the result code and a pre-formatted result label
+
+ Keyword arguments:
+ configurations (dictionary): a mapping of configuration key to value
+ parameters (dictionary): a mapping of script parameter key to value
+ host_name (string): the name of this host where the alert is running
+ """
+
+ if configurations is None:
+ return (UKNOWN_STATUS_CODE, ['There were no configurations supplied to the script.'])
+
+ result_code = None
+
+ try:
+ use_external_hbase = False
+ if USE_EXTERNAL_HBASE_KEY in configurations:
+ use_external_hbase = str(configurations[USE_EXTERNAL_HBASE_KEY]).upper() == 'TRUE'
+
+ if use_external_hbase:
+ return (OK_RESULT_CODE, ['use_external_hbase set to true.'])
+
+ is_hbase_system_service_launch = False
+ if ATS_HBASE_SYSTEM_SERVICE_LAUNCH_KEY in configurations:
+ is_hbase_system_service_launch = str(configurations[ATS_HBASE_SYSTEM_SERVICE_LAUNCH_KEY]).upper() == 'TRUE'
+
+ yarn_hbase_user = "yarn-ats"
+ if ATS_HBASE_USER_KEY in configurations:
+ yarn_hbase_user = configurations[ATS_HBASE_USER_KEY]
+
+ if not is_hbase_system_service_launch:
+ yarn_hbase_pid_dir_prefix = ""
+ if ATS_HBASE_PID_DIR_PREFIX in configurations:
+ yarn_hbase_pid_dir_prefix = configurations[ATS_HBASE_PID_DIR_PREFIX]
+ else:
+ return (UKNOWN_STATUS_CODE, ['The yarn_hbase_pid_dir_prefix is a required parameter.'])
+ yarn_hbase_pid_dir = format("{yarn_hbase_pid_dir_prefix}/{yarn_hbase_user}")
+ master_pid_file = format("{yarn_hbase_pid_dir}/hbase-{yarn_hbase_user}-master.pid")
+ rs_pid_file = format("{yarn_hbase_pid_dir}/hbase-{yarn_hbase_user}-regionserver.pid")
+
+ if host_name is None:
+ host_name = socket.getfqdn()
+
+ master_process_running = is_monitor_process_live(master_pid_file)
+ rs_process_running = is_monitor_process_live(rs_pid_file)
+
+ alert_state = OK_RESULT_CODE if master_process_running and rs_process_running else CRITICAL_RESULT_CODE
+
+ alert_label = 'ATS embedded HBase is running on {0}' if master_process_running and rs_process_running else 'ATS embedded HBase is NOT running on {0}'
+ alert_label = alert_label.format(host_name)
+
+ return (alert_state, [alert_label])
+ else:
+ security_enabled = False
+ if SECURITY_ENABLED_KEY in configurations:
+ security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == 'TRUE'
+
+ check_command_timeout = CHECK_COMMAND_TIMEOUT_DEFAULT
+ if CHECK_COMMAND_TIMEOUT_KEY in configurations:
+ check_command_timeout = int(parameters[CHECK_COMMAND_TIMEOUT_KEY])
+
+ if security_enabled:
+ if ATS_HBASE_PRINCIPAL_KEY in configurations:
+ ats_hbase_app_principal = configurations[ATS_HBASE_PRINCIPAL_KEY]
+ ats_hbase_app_principal = ats_hbase_app_principal.replace('_HOST',host_name.lower())
+
+ if ATS_HBASE_PRINCIPAL_KEYTAB_KEY in configurations:
+ ats_hbase_app_keytab = configurations[ATS_HBASE_PRINCIPAL_KEYTAB_KEY]
+
+ # Get the configured Kerberos executable search paths, if any
+ if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
+ kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
+ else:
+ kerberos_executable_search_paths = None
+
+ kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
+ kinitcmd=format("{kinit_path_local} -kt {ats_hbase_app_keytab} {ats_hbase_app_principal}; ")
+
+ # prevent concurrent kinit
+ kinit_lock = global_lock.get_lock(global_lock.LOCK_TYPE_KERBEROS)
+ kinit_lock.acquire()
+ try:
+ Execute(kinitcmd, user=yarn_hbase_user,
+ path=["/bin/", "/usr/bin/", "/usr/sbin/"],
+ timeout=10)
+ finally:
+ kinit_lock.release()
+
+ start_time = time.time()
+ ats_hbase_status_cmd = STACK_ROOT_DEFAULT + format("/current/hadoop-yarn-client/bin/yarn app -status ats-hbase")
+
+ code, output, error = shell.checked_call(ats_hbase_status_cmd, user=yarn_hbase_user, stderr=subprocess.PIPE,
+ timeout=check_command_timeout,
+ logoutput=False)
+ if code != 0:
+ alert_label = traceback.format_exc()
+ result_code = UKNOWN_STATUS_CODE
+ return (result_code, [alert_label])
+
+
+ # Call for getting JSON
+ ats_hbase_app_info = make_valid_json(output)
+
+ if ats_hbase_app_info is None:
+ alert_label = CRITICAL_MESSAGE
+ result_code = CRITICAL_RESULT_CODE
+ return (result_code, [alert_label])
+
+ if 'state' not in ats_hbase_app_info:
+ alert_label = traceback.format_exc()
+ result_code = UKNOWN_STATUS_CODE
+ return (result_code, [alert_label])
+
+ retrieved_ats_hbase_app_state = ats_hbase_app_info['state'].upper()
+
+ if retrieved_ats_hbase_app_state in ['STABLE']:
+ result_code = OK_RESULT_CODE
+ total_time = time.time() - start_time
+ alert_label = OK_MESSAGE.format(retrieved_ats_hbase_app_state, total_time)
+ else:
+ result_code = CRITICAL_RESULT_CODE
+ total_time = time.time() - start_time
+ alert_label = CRITICAL_MESSAGE_WITH_STATE.format(retrieved_ats_hbase_app_state, total_time)
+ except:
+ alert_label = traceback.format_exc()
+ traceback.format_exc()
+ result_code = CRITICAL_RESULT_CODE
+ return (result_code, [alert_label])
+
+
+def make_valid_json(output):
+ splits = output.split("\n")
+ ats_hbase_app_info = None
+ json_element = None # To detect where from to start reading for JSON data
+ for idx, split in enumerate(splits):
+ curr_elem = split.strip()
+ if curr_elem.startswith( '{' ) and curr_elem.endswith( '}' ):
+ json_element = curr_elem
+ break
+ elif ATS_HBASE_APP_NOT_FOUND_KEY in curr_elem:
+ return ats_hbase_app_info
+
+ # Remove extra logging from possible JSON output
+ if json_element is None:
+ raise Fail("Couldn't validate the received output for JSON parsing.")
+
+ ats_hbase_app_info = json.loads(json_element)
+ return ats_hbase_app_info
+
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def is_monitor_process_live(pid_file):
+ """
+ Gets whether the Metrics Monitor represented by the specified file is running.
+ :param pid_file: the PID file of the monitor to check
+ :return: True if the monitor is running, False otherwise
+ """
+ live = False
+
+ try:
+ check_process_status(pid_file)
+ live = True
+ except ComponentIsNotRunning:
+ pass
+
+ return live
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/functions.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/functions.py
new file mode 100755
index 0000000000..c436c62ac3
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/functions.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import re
+import math
+
+def calc_heap_memory(memorysize, heapmemory_factor):
+ """
+ @param memorysize_str: str (e.g '4096m')
+ @param heapmemory_factor: float (e.g 0.8)
+ """
+ return int(math.floor(memorysize*heapmemory_factor))
+
+def ensure_unit_for_memory(memory_size):
+ memory_size_values = re.findall('\d+', str(memory_size))
+ memory_size_unit = re.findall('\D+', str(memory_size))
+
+ if len(memory_size_values) > 0:
+ unit = 'm'
+ if len(memory_size_unit) > 0:
+ unit = memory_size_unit[0]
+ if unit not in ['b', 'k', 'm', 'g', 't', 'p']:
+ raise Exception("Memory size unit error. %s - wrong unit" % unit)
+ return "%s%s" % (memory_size_values[0], unit)
+ else:
+ raise Exception('Memory size can not be calculated')
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/hbase_service.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/hbase_service.py
new file mode 100755
index 0000000000..8e9e69a2ea
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/hbase_service.py
@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.show_logs import show_logs
+from resource_management.core.shell import as_sudo
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import Template
+from resource_management.core.logger import Logger
+
+def hbase_service(
+ name,
+ action = 'start'): # 'start' or 'stop'
+
+ import params
+
+ sudo = AMBARI_SUDO_BINARY
+ daemon_script = format("{yarn_hbase_bin}/hbase-daemon.sh")
+ role = name
+ cmd = format("{daemon_script} --config {yarn_hbase_conf_dir}")
+ pid_file = format("{yarn_hbase_pid_dir}/hbase-{yarn_hbase_user}-{role}.pid")
+ pid_expression = as_sudo(["cat", pid_file])
+ no_op_test = as_sudo(["test", "-f", pid_file]) + format(" && ps -p `{pid_expression}` >/dev/null 2>&1")
+
+ if action == 'start':
+ daemon_cmd = format("{cmd} start {role}")
+
+ try:
+ Execute ( daemon_cmd,
+ not_if = no_op_test,
+ user = params.yarn_hbase_user
+ )
+ except:
+ show_logs(params.yarn_hbase_log_dir, params.yarn_hbase_user)
+ raise
+ elif action == 'stop':
+ daemon_cmd = format("{cmd} stop {role}")
+
+ try:
+ Execute ( daemon_cmd,
+ user = params.yarn_hbase_user,
+ only_if = no_op_test,
+ timeout = 30,
+ on_timeout = format("! ( {no_op_test} ) || {sudo} -H -E kill -9 `{pid_expression}`"),
+ )
+ except:
+ show_logs(params.yarn_hbase_log_dir, params.yarn_hbase_user)
+ raise
+
+ File(pid_file,
+ action = "delete",
+ )
+
+def hbase(action):
+ if action == 'stop':
+ Logger.info("Stopping HBase daemons")
+ hbase_service('regionserver', action=action)
+ hbase_service('master', action=action)
+ else:
+ Logger.info("Starting HBase daemons")
+ hbase_service('master', action=action)
+ hbase_service('regionserver', action=action)
+ createTables()
+
+
+def configure_hbase(env):
+ import params
+ env.set_params(params)
+ params.HdfsResource(params.yarn_hbase_hdfs_root_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.yarn_hbase_user
+ )
+ params.HdfsResource(None, action="execute")
+
+def create_hbase_package():
+ import params
+ file_path = format("{yarn_hbase_package_preparation_file}")
+ Logger.info("Executing hbase package creation script file '" + file_path +"'")
+ try:
+ File( file_path,
+ mode = 0o755,
+ content = Template('yarn_hbase_package_preparation.j2')
+ )
+ Execute( file_path,
+ timeout = 300,
+ logoutput = True
+ )
+ except:
+ Logger.error(
+ "Error occured while executing hbase package creation file '" + file_path + "'.")
+ raise
+
+def copy_hbase_package_to_hdfs():
+ import params
+
+ try:
+
+ Logger.info(
+ "Copying hbase tarball into hdfs path'" + params.yarn_hbase_app_hdfs_path + "'.")
+ params.HdfsResource(format("{yarn_hbase_app_hdfs_path}"),
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ group=params.hdfs_user,
+ mode=0o555,
+ )
+ params.HdfsResource(format("{yarn_hbase_app_hdfs_path}/hbase.tar.gz"),
+ type="file",
+ action="create_on_execute",
+ source=format("{yarn_hbase_user_tmp}/hbase.tar.gz"),
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0o444,
+ )
+ params.HdfsResource(None, action="execute")
+ except:
+ Logger.error(
+ "Error occured while copying hbase tarball into hdfs '" + params.yarn_hbase_app_hdfs_path + "'.")
+ raise
+
+
+def createTables():
+ import params
+ try:
+ Logger.info("Creating HBase tables")
+ Execute(format("sleep 10;{yarn_hbase_table_create_cmd}"),
+ user=params.yarn_hbase_user,
+ timeout = 300,
+ logoutput=True)
+ except:
+ show_logs(params.yarn_hbase_log_dir, params.yarn_hbase_user)
+ raise
+
+ if params.security_enabled:
+ try:
+ File( format("{yarn_hbase_grant_premissions_file}"),
+ owner = params.yarn_hbase_user,
+ group = params.user_group,
+ mode = 0o644,
+ content = Template('yarn_hbase_grant_permissions.j2')
+ )
+ Execute( format("{yarn_hbase_table_grant_premission_cmd}"),
+ user = params.yarn_hbase_user,
+ timeout = 300,
+ logoutput = True
+ )
+ except:
+ show_logs(params.yarn_hbase_log_dir, params.yarn_hbase_user)
+ raise
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py
index 4217990f95..9846b95954 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py
@@ -76,12 +76,34 @@ class HistoryServerDefault(HistoryServer):
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
stack_select.select_packages(params.version)
+ # MC Hammer said, "Can't touch this"
+ copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
+ copy_to_hdfs("tez", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
+ params.HdfsResource(None, action="execute")
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env) # FOR SECURITY
+ if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major):
+ # MC Hammer said, "Can't touch this"
+ resource_created = copy_to_hdfs(
+ "mapreduce",
+ params.user_group,
+ params.hdfs_user,
+ skip=params.sysprep_skip_copy_tarballs_hdfs)
+ resource_created = copy_to_hdfs(
+ "tez",
+ params.user_group,
+ params.hdfs_user,
+ skip=params.sysprep_skip_copy_tarballs_hdfs) or resource_created
+ if resource_created:
+ params.HdfsResource(None, action="execute")
+ else:
+ # In stack versions before copy_tarball_to_hdfs support tez.tar.gz was copied to a different folder in HDFS.
+ install_tez_jars()
+
service('historyserver', action='start', serviceName='mapreduce')
def status(self, env):
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py
index ef2d437b41..a61701de18 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/params_linux.py
@@ -36,12 +36,15 @@ from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries.functions.version import format_stack_version, get_major_version
from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
from resource_management.libraries import functions
from resource_management.libraries.functions import is_empty
from resource_management.libraries.functions.get_architecture import get_architecture
from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs, generate_ranger_service_config
import status_params
+from functions import calc_heap_memory, ensure_unit_for_memory
+
service_name = 'yarn'
# a map of the Ambari role to the component name
@@ -55,7 +58,9 @@ YARN_SERVER_ROLE_DIRECTORY_MAP = {
'APP_TIMELINE_SERVER' : 'hadoop-yarn-timelineserver',
'NODEMANAGER' : 'hadoop-yarn-nodemanager',
'RESOURCEMANAGER' : 'hadoop-yarn-resourcemanager',
- 'YARN_CLIENT' : 'hadoop-yarn-client'
+ 'YARN_CLIENT' : 'hadoop-yarn-client',
+ 'TIMELINE_READER' : 'hadoop-yarn-timelinereader',
+ 'YARN_REGISTRY_DNS' : 'hadoop-yarn-registrydns'
}
# server configurations
@@ -152,6 +157,12 @@ yarn_bin = format("{hadoop_yarn_home}/sbin")
yarn_container_bin = format("{hadoop_yarn_home}/bin")
hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
+# MapR directory root
+mapred_role_root = "hadoop-mapreduce-client"
+command_role = default("/role", "")
+if command_role in MAPR_SERVER_ROLE_DIRECTORY_MAP:
+ mapred_role_root = MAPR_SERVER_ROLE_DIRECTORY_MAP[command_role]
+
if stack_supports_timeline_state_store:
# Timeline Service property that was added timeline_state_store stack feature
@@ -191,6 +202,8 @@ current_nm_security_state = os.path.isfile(nm_security_marker)
toggle_nm_security = (current_nm_security_state and not security_enabled) or (not current_nm_security_state and security_enabled)
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+mapred2_service_check_test_file = format('{tmp_dir}/mapred2-service-check')
+
yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group']
yarn_nodemanager_container_executor_class = config['configurations']['yarn-site']['yarn.nodemanager.container-executor.class']
is_linux_container_executor = (yarn_nodemanager_container_executor_class == 'org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor')
@@ -209,6 +222,7 @@ rm_nodes_exclude_dir = os.path.dirname(rm_nodes_exclude_path)
java64_home = config['ambariLevelParams']['java_home']
java_exec = format("{java64_home}/bin/java")
hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False)
+java_version = expect("/ambariLevelParams/java_version", int)
yarn_heapsize = config['configurations']['yarn-env']['yarn_heapsize']
resourcemanager_heapsize = config['configurations']['yarn-env']['resourcemanager_heapsize']
@@ -292,6 +306,12 @@ if include_file_path and manage_include_files:
ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
has_ats = not len(ats_host) == 0
+atsv2_host = set(default("/clusterHostInfo/timeline_reader_hosts", []))
+has_atsv2 = not len(atsv2_host) == 0
+
+registry_dns_host = set(default("/clusterHostInfo/yarn_registry_dns_hosts", []))
+has_registry_dns = not len(registry_dns_host) == 0
+
# don't using len(nm_hosts) here, because check can take too much time on large clusters
number_of_nm = 1
@@ -322,13 +342,19 @@ if security_enabled:
rm_security_opts = format('-Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username={zk_principal_user} -Djava.security.auth.login.config={yarn_jaas_file} -Dzookeeper.sasl.clientconfig=Client')
# YARN timeline security options
- if has_ats:
+ if has_ats or has_atsv2:
yarn_timelineservice_principal_name = config['configurations']['yarn-site']['yarn.timeline-service.principal']
yarn_timelineservice_principal_name = yarn_timelineservice_principal_name.replace('_HOST', hostname.lower())
yarn_timelineservice_keytab = config['configurations']['yarn-site']['yarn.timeline-service.keytab']
yarn_timelineservice_kinit_cmd = format("{kinit_path_local} -kt {yarn_timelineservice_keytab} {yarn_timelineservice_principal_name};")
yarn_ats_jaas_file = os.path.join(hadoop_conf_dir, 'yarn_ats_jaas.conf')
+ if has_registry_dns:
+ yarn_registry_dns_principal_name = config['configurations']['yarn-env']['yarn.registry-dns.principal']
+ yarn_registry_dns_principal_name = yarn_registry_dns_principal_name.replace('_HOST', hostname.lower())
+ yarn_registry_dns_keytab = config['configurations']['yarn-env']['yarn.registry-dns.keytab']
+ yarn_registry_dns_jaas_file = os.path.join(hadoop_conf_dir, 'yarn_registry_dns_jaas.conf')
+
if 'yarn.nodemanager.principal' in config['configurations']['yarn-site']:
nodemanager_principal_name = default('/configurations/yarn-site/yarn.nodemanager.principal', None)
if nodemanager_principal_name:
@@ -421,6 +447,7 @@ yarn_rm_address = config['configurations']['yarn-site']['yarn.resourcemanager.we
rm_active_port = rm_https_port if yarn_https_on else rm_port
rm_ha_enabled = False
+rm_ha_id = None
rm_ha_ids_list = []
rm_webapp_addresses_list = [yarn_rm_address]
rm_ha_ids = default("/configurations/yarn-site/yarn.resourcemanager.ha.rm-ids", None)
@@ -436,6 +463,9 @@ if rm_ha_enabled:
rm_webapp_address_property = format('yarn.resourcemanager.webapp.address.{rm_id}') if not yarn_https_on else format('yarn.resourcemanager.webapp.https.address.{rm_id}')
rm_webapp_address = config['configurations']['yarn-site'][rm_webapp_address_property]
rm_webapp_addresses_list.append(rm_webapp_address)
+ rm_host_name = config['configurations']['yarn-site'][format('yarn.resourcemanager.hostname.{rm_id}')]
+ if rm_host_name == hostname.lower():
+ rm_ha_id = rm_id
# for curl command in ranger plugin to get db connector
jdk_location = config['ambariLevelParams']['jdk_location']
@@ -574,3 +604,223 @@ if enable_ranger_yarn and is_supported_yarn_ranger:
# need this to capture cluster name from where ranger yarn plugin is enabled
cluster_name = config['clusterName']
+
+# ranger yarn plugin end section
+
+# container-executor properties
+min_user_id = config['configurations']['container-executor']['min_user_id']
+docker_module_enabled = str(config['configurations']['container-executor']['docker_module_enabled']).lower()
+docker_binary = config['configurations']['container-executor']['docker_binary']
+docker_allowed_capabilities = config['configurations']['yarn-site']['yarn.nodemanager.runtime.linux.docker.capabilities']
+if docker_allowed_capabilities:
+ docker_allowed_capabilities = ','.join(x.strip() for x in docker_allowed_capabilities.split(','))
+else:
+ docker_allowed_capabilities = ""
+docker_allowed_devices = config['configurations']['container-executor']['docker_allowed_devices']
+docker_allowed_networks = config['configurations']['yarn-site']['yarn.nodemanager.runtime.linux.docker.allowed-container-networks']
+if docker_allowed_networks:
+ docker_allowed_networks = ','.join(x.strip() for x in docker_allowed_networks.split(','))
+else:
+ docker_allowed_networks = ""
+docker_allowed_ro_mounts = config['configurations']['container-executor']['docker_allowed_ro-mounts']
+docker_allowed_rw_mounts = config['configurations']['container-executor']['docker_allowed_rw-mounts']
+docker_privileged_containers_enabled = str(config['configurations']['container-executor']['docker_privileged-containers_enabled']).lower()
+docker_trusted_registries = config['configurations']['container-executor']['docker_trusted_registries']
+docker_allowed_volume_drivers = config['configurations']['container-executor']['docker_allowed_volume-drivers']
+
+# ATSv2 integration properties started.
+yarn_timelinereader_pid_file = status_params.yarn_timelinereader_pid_file
+
+yarn_atsv2_hbase_versioned_home = format("{stack_root}/{version}/usr/lib/hbase")
+yarn_hbase_bin = format("{yarn_atsv2_hbase_versioned_home}/bin")
+yarn_hbase_hdfs_root_dir = config['configurations']['yarn-hbase-site']['hbase.rootdir']
+cluster_zookeeper_quorum_hosts = ",".join(config['clusterHostInfo']['zookeeper_server_hosts'])
+if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+ cluster_zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+else:
+ cluster_zookeeper_clientPort = '2181'
+
+zookeeper_quorum_hosts = cluster_zookeeper_quorum_hosts
+zookeeper_clientPort = cluster_zookeeper_clientPort
+yarn_hbase_user = status_params.yarn_hbase_user
+hbase_user = config['configurations']['hbase-env']['hbase_user']
+yarn_hbase_user_home = format("/user/{yarn_hbase_user}")
+yarn_hbase_user_version_home = format("{yarn_hbase_user_home}/{version}")
+yarn_hbase_app_hdfs_path = format("/bigtop/apps/{version}/hbase")
+yarn_service_app_hdfs_path = format("/bigtop/apps/{version}/yarn")
+if rm_ha_id is not None:
+ yarn_hbase_app_hdfs_path = format("{yarn_hbase_app_hdfs_path}/{rm_ha_id}")
+ yarn_service_app_hdfs_path = format("{yarn_service_app_hdfs_path}/{rm_ha_id}")
+yarn_service_dep_source_path = format("{stack_root}/{version}/usr/lib/hadoop-yarn/lib/service-dep.tar.gz")
+yarn_hbase_user_version_path = format("{yarn_hbase_user}/{version}")
+yarn_hbase_user_tmp = format("{tmp_dir}/{yarn_hbase_user_version_path}")
+yarn_hbase_log_dir = os.path.join(yarn_log_dir_prefix, "embedded-yarn-ats-hbase")
+yarn_hbase_pid_dir_prefix = status_params.yarn_hbase_pid_dir_prefix
+yarn_hbase_pid_dir = status_params.yarn_hbase_pid_dir
+yarn_hbase_conf_dir = os.path.join(hadoop_conf_dir, "embedded-yarn-ats-hbase")
+yarn_hbase_env_sh_template = config['configurations']['yarn-hbase-env']['content']
+yarn_hbase_java_io_tmpdir = default("/configurations/yarn-hbase-env/hbase_java_io_tmpdir", "/tmp")
+yarn_hbase_tmp_dir = config['configurations']['yarn-hbase-site']['hbase.tmp.dir']
+yarn_hbase_local_dir = config['configurations']['yarn-hbase-site']['hbase.local.dir']
+yarn_hbase_master_info_port = config['configurations']['yarn-hbase-site']['hbase.master.info.port']
+yarn_hbase_regionserver_info_port = config['configurations']['yarn-hbase-site']['hbase.regionserver.info.port']
+
+if (('yarn-hbase-log4j' in config['configurations']) and ('content' in config['configurations']['yarn-hbase-log4j'])):
+ yarn_hbase_log4j_props = config['configurations']['yarn-hbase-log4j']['content']
+else:
+ yarn_hbase_log4j_props = None
+
+timeline_collector = ""
+yarn_timeline_service_version = config['configurations']['yarn-site']['yarn.timeline-service.version']
+yarn_timeline_service_versions = config['configurations']['yarn-site']['yarn.timeline-service.versions']
+yarn_timeline_service_enabled = config['configurations']['yarn-site']['yarn.timeline-service.enabled']
+
+if yarn_timeline_service_enabled:
+ if is_empty(yarn_timeline_service_versions):
+ if yarn_timeline_service_version == '2.0' or yarn_timeline_service_version == '2':
+ timeline_collector = "timeline_collector"
+ else:
+ ts_version_list = yarn_timeline_service_versions.split(',')
+ for ts_version in ts_version_list:
+ if '2.0' in ts_version or ts_version == '2':
+ timeline_collector = "timeline_collector"
+ break
+
+coprocessor_jar_name = "hadoop-yarn-server-timelineservice-hbase-coprocessor.jar"
+yarn_timeline_jar_location = format("file://{stack_root}/{version}/usr/lib/hadoop-yarn/timelineservice/{coprocessor_jar_name}")
+yarn_user_hbase_permissions = "RWXCA"
+
+yarn_hbase_kinit_cmd = ""
+if security_enabled and has_atsv2:
+ yarn_hbase_jaas_file = os.path.join(yarn_hbase_conf_dir, 'yarn_hbase_jaas.conf')
+ yarn_hbase_master_jaas_file = os.path.join(yarn_hbase_conf_dir, 'yarn_hbase_master_jaas.conf')
+ yarn_hbase_regionserver_jaas_file = os.path.join(yarn_hbase_conf_dir, 'yarn_hbase_regionserver_jaas.conf')
+
+ yarn_hbase_master_principal_name = config['configurations']['yarn-hbase-site']['hbase.master.kerberos.principal']
+ yarn_hbase_master_principal_name = yarn_hbase_master_principal_name.replace('_HOST', hostname.lower())
+ yarn_hbase_master_keytab = config['configurations']['yarn-hbase-site']['hbase.master.keytab.file']
+
+ yarn_hbase_regionserver_principal_name = config['configurations']['yarn-hbase-site']['hbase.regionserver.kerberos.principal']
+ yarn_hbase_regionserver_principal_name = yarn_hbase_regionserver_principal_name.replace('_HOST', hostname.lower())
+ yarn_hbase_regionserver_keytab = config['configurations']['yarn-hbase-site']['hbase.regionserver.keytab.file']
+
+ # User master principal name as AM principal in system service. Don't replace _HOST.
+ yarn_ats_hbase_principal_name = config['configurations']['yarn-hbase-site']['hbase.master.kerberos.principal']
+ yarn_ats_hbase_keytab = config['configurations']['yarn-hbase-site']['hbase.master.keytab.file']
+ yarn_ats_principal_name = config['configurations']['yarn-env']['yarn_ats_principal_name']
+ yarn_ats_user_keytab = config['configurations']['yarn-env']['yarn_ats_user_keytab']
+ yarn_hbase_kinit_cmd = format("{kinit_path_local} -kt {yarn_ats_user_keytab} {yarn_ats_principal_name};")
+
+
+hbase_within_cluster = config['configurations']['yarn-hbase-env']['hbase_within_cluster']
+is_hbase_installed = False
+master_configs = config['clusterHostInfo']
+
+if hbase_within_cluster:
+ if 'hbase_master_hosts' in master_configs and 'hbase-site' in config['configurations']:
+ is_hbase_installed = True
+ zookeeper_znode_parent = config['configurations']['hbase-site']['zookeeper.znode.parent']
+ else:
+ zookeeper_znode_parent = "/hbase-unsecure"
+ hbase_site_conf = config['configurations']['hbase-site']
+ hbase_site_attributes = config['configurationAttributes']['hbase-site']
+ yarn_hbase_conf_dir = "/etc/hbase/conf"
+else:
+ zookeeper_znode_parent = "/atsv2-hbase-unsecure"
+ hbase_site_conf = config['configurations']['yarn-hbase-site']
+ hbase_site_attributes = config['configurationAttributes']['yarn-hbase-site']
+
+yarn_hbase_grant_premissions_file = format("{yarn_hbase_conf_dir}/hbase_grant_permissions.sh")
+yarn_hbase_package_preparation_file = format("{tmp_dir}/hbase_package_preparation.sh")
+is_hbase_system_service_launch = config['configurations']['yarn-hbase-env']['is_hbase_system_service_launch']
+use_external_hbase = config['configurations']['yarn-hbase-env']['use_external_hbase']
+
+hbase_cmd = format("{yarn_hbase_bin}/hbase --config {yarn_hbase_conf_dir}")
+class_name = format("org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator -Dhbase.client.retries.number=35 -create -s")
+yarn_hbase_table_create_cmd = format("export HBASE_CLASSPATH_PREFIX={stack_root}/{version}/usr/lib/hadoop-yarn/timelineservice/*;{yarn_hbase_kinit_cmd} {hbase_cmd} {class_name}")
+yarn_hbase_table_grant_premission_cmd = format("{yarn_hbase_kinit_cmd} {hbase_cmd} shell {yarn_hbase_grant_premissions_file}")
+
+# System service configuration as part of ATSv2.
+yarn_system_service_dir = config['configurations']['yarn-site']['yarn.service.system-service.dir']
+yarn_system_service_launch_mode = config['configurations']['yarn-hbase-env']['yarn_hbase_system_service_launch_mode']
+yarn_hbase_service_queue_name = config['configurations']['yarn-hbase-env']['yarn_hbase_system_service_queue_name']
+
+yarn_hbase_master_cpu = config['configurations']['yarn-hbase-env']['yarn_hbase_master_cpu']
+yarn_hbase_master_memory = expect("/configurations/yarn-hbase-env/yarn_hbase_master_memory", int)
+yarn_hbase_master_containers = config['configurations']['yarn-hbase-env']['yarn_hbase_master_containers']
+yarn_hbase_regionserver_cpu = config['configurations']['yarn-hbase-env']['yarn_hbase_regionserver_cpu']
+yarn_hbase_regionserver_memory = expect("/configurations/yarn-hbase-env/yarn_hbase_regionserver_memory", int)
+yarn_hbase_regionserver_containers = config['configurations']['yarn-hbase-env']['yarn_hbase_regionserver_containers']
+yarn_hbase_client_cpu = config['configurations']['yarn-hbase-env']['yarn_hbase_client_cpu']
+yarn_hbase_client_memory = expect("/configurations/yarn-hbase-env/yarn_hbase_client_memory", int)
+yarn_hbase_client_containers = config['configurations']['yarn-hbase-env']['yarn_hbase_client_containers']
+
+yarn_hbase_heap_memory_factor = expect("/configurations/yarn-hbase-env/yarn_hbase_heap_memory_factor", float)
+yarn_hbase_master_heapsize = ensure_unit_for_memory(calc_heap_memory(yarn_hbase_master_memory, yarn_hbase_heap_memory_factor))
+yarn_hbase_regionserver_heapsize = ensure_unit_for_memory(calc_heap_memory(yarn_hbase_regionserver_memory, yarn_hbase_heap_memory_factor))
+
+yarn_hbase_log_level = str(config['configurations']['yarn-hbase-env']['yarn_hbase_log_level']).upper()
+# ATSv2 integration properties ended
+
+gpu_module_enabled = str(config['configurations']['container-executor']['gpu_module_enabled']).lower()
+cgroup_root = config['configurations']['container-executor']['cgroup_root']
+yarn_hierarchy = config['configurations']['container-executor']['yarn_hierarchy']
+
+# registry dns service
+registry_dns_needs_privileged_access = status_params.registry_dns_needs_privileged_access
+
+mount_table_content = None
+if 'viewfs-mount-table' in config['configurations']:
+ xml_inclusion_file_name = 'viewfs-mount-table.xml'
+ mount_table = config['configurations']['viewfs-mount-table']
+
+ if 'content' in mount_table and mount_table['content'].strip():
+ mount_table_content = mount_table['content']
+
+hbase_log_maxfilesize = default('configurations/yarn-hbase-log4j/hbase_log_maxfilesize',256)
+hbase_log_maxbackupindex = default('configurations/yarn-hbase-log4j/hbase_log_maxbackupindex',20)
+hbase_security_log_maxfilesize = default('configurations/yarn-hbase-log4j/hbase_security_log_maxfilesize',256)
+hbase_security_log_maxbackupindex = default('configurations/yarn-hbase-log4j/hbase_security_log_maxbackupindex',20)
+
+rm_cross_origin_enabled = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.cross-origin.enabled']
+
+cross_origins = '*'
+if rm_cross_origin_enabled:
+ host_suffix = rm_host.rsplit('.', 2)[1:]
+ if len(host_suffix) == 2 :
+ cross_origins = 'regex:.*[.]' + '[.]'.join(host_suffix) + "(:\d*)?"
+
+ams_collector_hosts = ",".join(default("/clusterHostInfo/metrics_collector_hosts", []))
+has_metric_collector = not len(ams_collector_hosts) == 0
+if has_metric_collector:
+ if 'cluster-env' in config['configurations'] and \
+ 'metrics_collector_vip_port' in config['configurations']['cluster-env']:
+ metric_collector_port = config['configurations']['cluster-env']['metrics_collector_vip_port']
+ else:
+ metric_collector_web_address = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
+ if metric_collector_web_address.find(':') != -1:
+ metric_collector_port = metric_collector_web_address.split(':')[1]
+ else:
+ metric_collector_port = '6188'
+ if default("/configurations/ams-site/timeline.metrics.service.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ metric_collector_protocol = 'https'
+ else:
+ metric_collector_protocol = 'http'
+ metric_truststore_path= default("/configurations/ams-ssl-client/ssl.client.truststore.location", "")
+ metric_truststore_type= default("/configurations/ams-ssl-client/ssl.client.truststore.type", "")
+ metric_truststore_password= default("/configurations/ams-ssl-client/ssl.client.truststore.password", "")
+ host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True)
+ host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888)
+
+ pass
+metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sink.report.interval", 60)
+metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
+
+host_in_memory_aggregation = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation", True)
+host_in_memory_aggregation_port = default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.port", 61888)
+is_aggregation_https_enabled = False
+if default("/configurations/ams-site/timeline.metrics.host.inmemory.aggregation.http.policy", "HTTP_ONLY") == "HTTPS_ONLY":
+ host_in_memory_aggregation_protocol = 'https'
+ is_aggregation_https_enabled = True
+else:
+ host_in_memory_aggregation_protocol = 'http'
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/service.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/service.py
index 276bad38f6..5e2e63e71f 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/service.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/service.py
@@ -22,12 +22,14 @@ Ambari Agent
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
+from resource_management.core import shell
from resource_management.core.shell import as_user, as_sudo
+from resource_management.core.logger import Logger
from resource_management.libraries.functions.show_logs import show_logs
from resource_management.libraries.functions.format import format
from resource_management.core.resources.system import Execute, File
from resource_management.core.signal_utils import TerminateStrategy
-from resource_management.core.logger import Logger
+import subprocess
@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
def service(componentName, action='start', serviceName='yarn'):
@@ -43,34 +45,58 @@ def service(componentName, action='start', serviceName='yarn'):
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def service(componentName, action='start', serviceName='yarn'):
import params
+ import status_params
- if serviceName == 'mapreduce' and componentName == 'historyserver':
- if not params.hdfs_tmp_dir or params.hdfs_tmp_dir == None or params.hdfs_tmp_dir.lower() == 'null':
- Logger.error("WARNING: HDFS tmp dir property (hdfs_tmp_dir) is empty or invalid. Ambari will change permissions for the folder on regular basis.")
+ hadoop_env_exports = {
+ 'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir
+ }
+ if serviceName == 'mapreduce' and componentName == 'historyserver':
delete_pid_file = True
daemon = format("{mapred_bin}/mr-jobhistory-daemon.sh")
pid_file = format("{mapred_pid_dir}/hadoop-{mapred_user}-{componentName}.pid")
usr = params.mapred_user
log_dir = params.mapred_log_dir
+ cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}")
else:
# !!! yarn-daemon.sh deletes the PID for us; if we remove it the script
# may not work correctly when stopping the service
delete_pid_file = False
daemon = format("{yarn_bin}/yarn-daemon.sh")
- pid_file = format("{yarn_pid_dir}/hadoop-{yarn_user}-{componentName}.pid")
- usr = params.yarn_user
+ if componentName == 'registrydns' and status_params.registry_dns_needs_privileged_access:
+ pid_file = status_params.yarn_registry_dns_priv_pid_file
+ usr = status_params.root_user
+ else:
+ pid_file = format("{yarn_pid_dir}/hadoop-{yarn_user}-{componentName}.pid")
+ usr = params.yarn_user
+
log_dir = params.yarn_log_dir
+ cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}")
- cmd = format("export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {daemon} --config {hadoop_conf_dir}")
+ check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
- if action == 'start':
+ if usr == 'root' and componentName == 'registrydns':
+ # these are needed for unknown reasons
+ hadoop_env_exports['HADOOP_PID_DIR'] = params.yarn_pid_dir
+ hadoop_env_exports['HADOOP_SECURE_PID_DIR'] = params.yarn_pid_dir
+ hadoop_env_exports['HADOOP_LOG_DIR'] = params.yarn_log_dir
+ hadoop_env_exports['HADOOP_SECURE_LOG_DIR'] = params.yarn_log_dir
- daemon_cmd = format("{ulimit_cmd} {cmd} start {componentName}")
- check_process = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
+ cmd = [daemon, "--config", params.hadoop_conf_dir, action, componentName]
+ daemon_cmd = as_sudo(cmd)
+ else:
+ if action == 'start':
+ cmd = format("{ulimit_cmd} {cmd} start {componentName}")
+ else:
+ cmd = format("{cmd} stop {componentName}")
+ daemon_cmd = as_user(cmd, usr)
- # Remove the pid file if its corresponding process is not running.
- File(pid_file, action = "delete", not_if = check_process)
+ if action == 'start':
+ if componentName == 'registrydns':
+ checkAndStopRegistyDNS()
+ else:
+ # Remove the pid file if its corresponding process is not running.
+ File(pid_file, action = "delete", not_if = check_process)
if componentName == 'timelineserver' and serviceName == 'yarn':
File(params.ats_leveldb_lock_file,
@@ -82,8 +108,8 @@ def service(componentName, action='start', serviceName='yarn'):
try:
# Attempt to start the process. Internally, this is skipped if the process is already running.
- Execute(daemon_cmd, user = usr, not_if = check_process)
-
+ Execute(daemon_cmd, not_if=check_process, environment=hadoop_env_exports)
+
# Ensure that the process with the expected PID exists.
Execute(check_process,
not_if = check_process,
@@ -95,22 +121,22 @@ def service(componentName, action='start', serviceName='yarn'):
raise
elif action == 'stop':
- daemon_cmd = format("{cmd} stop {componentName}")
- try:
- Execute(daemon_cmd, user=usr)
- except:
- show_logs(log_dir, usr)
- raise
-
- # !!! yarn-daemon doesn't need us to delete PIDs
- if delete_pid_file is True:
- File(pid_file, action="delete")
-
+ if componentName == 'registrydns':
+ checkAndStopRegistyDNS()
+ else:
+ try:
+ Execute(daemon_cmd, only_if=check_process, environment=hadoop_env_exports)
+ except:
+ show_logs(log_dir, usr)
+ raise
+
+ # !!! yarn-daemon doesn't need us to delete PIDs
+ if delete_pid_file is True:
+ File(pid_file, action="delete")
elif action == 'refreshQueues':
rm_kinit_cmd = params.rm_kinit_cmd
refresh_cmd = format("{rm_kinit_cmd} export HADOOP_LIBEXEC_DIR={hadoop_libexec_dir} && {yarn_container_bin}/yarn rmadmin -refreshQueues")
-
Execute(refresh_cmd,
user = usr,
timeout = 20, # when Yarn is not started command hangs forever and should be killed
@@ -118,3 +144,75 @@ def service(componentName, action='start', serviceName='yarn'):
try_sleep = 5,
timeout_kill_strategy = TerminateStrategy.KILL_PROCESS_GROUP, # the process cannot be simply killed by 'kill -15', so kill pg group instread.
)
+
+def checkAndStopRegistyDNS():
+ import params
+ import status_params
+
+ componentName = 'registrydns'
+ action = 'stop'
+ daemon = format("{yarn_bin}/yarn-daemon.sh")
+ hadoop_env_exports = {
+ 'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir
+ }
+
+ # When registry dns is switched from non-privileged to privileged mode or the other way,
+ # then the previous instance of registry dns has a different pid/user.
+ # Checking if either of the processes are running and shutting them down if they are.
+
+ # privileged mode
+ dns_pid_file = status_params.yarn_registry_dns_priv_pid_file
+ dns_user = status_params.root_user
+ Logger.info("checking any existing dns pid file = '" + dns_pid_file + "' dns user '" + dns_user + "'")
+ try:
+ # these are needed for unknown reasons
+ env_exports = {
+ 'HADOOP_PID_DIR': params.yarn_pid_dir,
+ 'HADOOP_SECURE_PID_DIR': params.yarn_pid_dir,
+ 'HADOOP_LOG_DIR': params.yarn_log_dir,
+ 'HADOOP_SECURE_LOG_DIR': params.yarn_log_dir
+ }
+ env_exports.update(hadoop_env_exports)
+ cmd = [daemon, "--config", params.hadoop_conf_dir, action, componentName]
+ daemon_cmd = as_sudo(cmd)
+ process_id_exists_command = as_sudo(["test", "-f", dns_pid_file]) + " && " + as_sudo(["pgrep", "-F", dns_pid_file])
+ Execute(daemon_cmd, only_if=process_id_exists_command, environment=env_exports)
+ except:
+ # When the registry dns port is modified but registry dns is not started
+ # immediately, then the configs in yarn-env.sh & yarn-site.xml related
+ # to registry dns may have already changed. This introduces a discrepancy
+ # between the actual process that is running and the configs.
+ # For example, when port is changed from 5300 to 53,
+ # then dns port = 53 in yarn-site and YARN_REGISTRYDNS_SECURE_* envs in yarn-env.sh
+ # are saved. So, while trying to shutdown the stray non-privileged registry dns process
+ # after sometime, yarn daemon from the configs thinks that it needs privileged
+ # access and throws an exception. In such cases, we try to kill the stray process.
+ pass
+ process_id_does_not_exist_command = format("! ( {process_id_exists_command} )")
+ code, out = shell.call(process_id_does_not_exist_command,
+ env=env_exports,
+ tries=5,
+ try_sleep=5)
+ if code != 0:
+ code, out, err = shell.checked_call(("pgrep", "-f", dns_pid_file), sudo=True, env=env_exports,
+ stderr=subprocess.PIPE)
+ Logger.info("PID to kill was retrieved: '" + out + "'.")
+ for pid in out.splitlines():
+ try:
+ Execute(("kill", "-9", pid), sudo=True)
+ except:
+ # ignoring failures
+ Logger.warning("failed to kill pid '" + pid + "'.")
+ pass
+ File(dns_pid_file, action="delete")
+
+ # non-privileged mode
+ dns_pid_file = status_params.yarn_registry_dns_pid_file
+ dns_user = params.yarn_user
+ Logger.info("checking any existing dns pid file = '" + dns_pid_file + "' dns user '" + dns_user + "'")
+ try:
+ cmd = format("{daemon} --config {hadoop_conf_dir} {action} {componentName}")
+ daemon_cmd = as_user(cmd, dns_user)
+ Execute(daemon_cmd, environment=hadoop_env_exports)
+ except:
+ pass
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/status_params.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/status_params.py
index aaace6cd86..e0528006b9 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/status_params.py
@@ -17,18 +17,27 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import os
+import pwd
+
+from ambari_commons import OSCheck
+
from resource_management.libraries.script.script import Script
from resource_management.libraries import functions
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions.default import default
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
+root_user = 'root'
mapred_user = config['configurations']['mapred-env']['mapred_user']
yarn_user = config['configurations']['yarn-env']['yarn_user']
+yarn_ats_user = config['configurations']['yarn-env']['yarn_ats_user']
yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix']
mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix']
yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}")
@@ -38,8 +47,23 @@ resourcemanager_pid_file = format("{yarn_pid_dir}/hadoop-{yarn_user}-resourceman
nodemanager_pid_file = format("{yarn_pid_dir}/hadoop-{yarn_user}-nodemanager.pid")
yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/hadoop-{yarn_user}-historyserver.pid")
yarn_historyserver_pid_file = format("{yarn_pid_dir}/hadoop-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated
+
+# registry dns service
+registry_dns_bind_port = int(config['configurations']['yarn-env']['registry.dns.bind-port'])
+registry_dns_needs_privileged_access = True if registry_dns_bind_port < 1024 else False
+
+yarn_registry_dns_pid_file = format("{yarn_pid_dir_prefix}/{yarn_user}/hadoop-{yarn_user}-registrydns.pid")
+yarn_registry_dns_priv_pid_file = format("{yarn_pid_dir_prefix}/{root_user}/hadoop-{yarn_user}-{root_user}-registrydns.pid")
+
+if registry_dns_needs_privileged_access:
+ yarn_registry_dns_in_use_pid_file = yarn_registry_dns_priv_pid_file
+else:
+ yarn_registry_dns_in_use_pid_file = yarn_registry_dns_pid_file
+
mapred_historyserver_pid_file = format("{mapred_pid_dir}/hadoop-{mapred_user}-historyserver.pid")
+yarn_timelinereader_pid_file = format("{yarn_pid_dir}/hadoop-{yarn_user}-timelinereader.pid")
+
hadoop_home = stack_select.get_hadoop_dir("home")
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
@@ -48,3 +72,8 @@ kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-en
security_enabled = config['configurations']['cluster-env']['security_enabled']
stack_name = default("/clusterLevelParams/stack_name", None)
+
+# ATSv2 backend properties
+yarn_hbase_user = format("{yarn_ats_user}") #Use yarn_ats_user.
+yarn_hbase_pid_dir_prefix = config['configurations']['yarn-hbase-env']['yarn_hbase_pid_dir_prefix']
+yarn_hbase_pid_dir = format("{yarn_hbase_pid_dir_prefix}/{yarn_hbase_user}")
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/timelinereader.py
old mode 100644
new mode 100755
similarity index 51%
copy from ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py
copy to ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/timelinereader.py
index 4217990f95..0ee3e2ae08
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/timelinereader.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -19,56 +18,68 @@ limitations under the License.
Ambari Agent
"""
+import os
from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions.check_process_status import check_process_status
-from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
-from resource_management.libraries.functions.security_commons import build_expectations, \
- cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
- FILE_TYPE_XML
-from resource_management.core.source import Template
+from resource_management.libraries.functions import check_process_status
+from resource_management.libraries.functions.format import format
from resource_management.core.logger import Logger
-from install_jars import install_tez_jars
from yarn import yarn
from service import service
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyImpl
+from hbase_service import hbase, configure_hbase
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
-
-class HistoryServer(Script):
+class ApplicationTimelineReader(Script):
def install(self, env):
self.install_packages(env)
- def stop(self, env, upgrade_type=None):
+ def start(self, env, upgrade_type=None):
import params
env.set_params(params)
- service('historyserver', action='stop', serviceName='mapreduce')
+ self.configure(env) # FOR SECURITY
+
+ if params.stack_version_formatted_major and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS, params.stack_version_formatted_major):
+ # MC Hammer said, "Can't touch this"
+ resource_created = copy_to_hdfs(
+ "yarn",
+ params.user_group,
+ params.hdfs_user,
+ skip=params.sysprep_skip_copy_tarballs_hdfs)
+ if resource_created:
+ params.HdfsResource(None, action="execute")
+
+ if not params.use_external_hbase and not params.is_hbase_system_service_launch:
+ hbase(action='start')
+ service('timelinereader', action='start')
- def configure(self, env):
+ def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
- yarn(name="historyserver")
-
+ if not params.use_external_hbase and not params.is_hbase_system_service_launch:
+ hbase(action='stop')
+ service('timelinereader', action='stop')
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class HistoryserverWindows(HistoryServer):
- def start(self, env):
+ def configure(self, env, action = None):
import params
env.set_params(params)
- self.configure(env)
- service('historyserver', action='start', serviceName='mapreduce')
+ yarn(name='apptimelinereader')
+ if not params.use_external_hbase and not params.is_hbase_system_service_launch:
+ configure_hbase(env)
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class ApplicationTimelineReaderWindows(ApplicationTimelineReader):
def status(self, env):
- service('historyserver', action='status')
+ service('timelinereader', action='status')
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class HistoryServerDefault(HistoryServer):
+class ApplicationTimelineReaderDefault(ApplicationTimelineReader):
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Stack Upgrade pre-restart")
import params
@@ -76,30 +87,33 @@ class HistoryServerDefault(HistoryServer):
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
stack_select.select_packages(params.version)
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- self.configure(env) # FOR SECURITY
-
- service('historyserver', action='start', serviceName='mapreduce')
+ # MC Hammer said, "Can't touch this"
+ resource_created = copy_to_hdfs("yarn", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
+ if resource_created:
+ params.HdfsResource(None, action="execute")
def status(self, env):
import status_params
env.set_params(status_params)
- check_process_status(status_params.mapred_historyserver_pid_file)
+ for pid_file in self.get_pid_files():
+ check_process_status(pid_file)
def get_log_folder(self):
import params
- return params.mapred_log_dir
+ return params.yarn_log_dir
def get_user(self):
import params
- return params.mapred_user
+ return params.yarn_user
def get_pid_files(self):
- import status_params
- return [status_params.mapred_historyserver_pid_file]
+ import params
+ pid_files = []
+ pid_files.append(format("{yarn_timelinereader_pid_file}"))
+ if not params.use_external_hbase and not params.is_hbase_system_service_launch:
+ pid_files.append(format("{yarn_hbase_pid_dir}/hbase-{yarn_hbase_user}-master.pid"))
+ pid_files.append(format("{yarn_hbase_pid_dir}/hbase-{yarn_hbase_user}-regionserver.pid"))
+ return pid_files
if __name__ == "__main__":
- HistoryServer().execute()
+ ApplicationTimelineReader().execute()
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/yarn.py
index 2c6ee037ec..db0d75003a 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/yarn.py
@@ -29,15 +29,17 @@ from resource_management.core.resources.service import ServiceConfig
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.is_empty import is_empty
from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
-from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import Directory,Execute
from resource_management.core.resources.system import File
from resource_management.libraries.resources.xml_config import XmlConfig
from resource_management.core.source import InlineTemplate, Template
from resource_management.core.logger import Logger
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
from resource_management.libraries.functions.mounted_dirs_helper import handle_mounted_dirs
+from hbase_service import create_hbase_package, copy_hbase_package_to_hdfs, createTables
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def yarn(name=None, config_dir=None):
@@ -80,6 +82,12 @@ def yarn(name=None, config_dir=None):
ignore_failures=True,
cd_access='a',
)
+ Directory(params.yarn_hbase_conf_dir,
+ owner = params.yarn_hbase_user,
+ group = params.user_group,
+ create_parents = True,
+ cd_access='a',
+ )
# Some of these function calls depend on the directories above being created first.
if name == 'resourcemanager':
@@ -90,6 +98,10 @@ def yarn(name=None, config_dir=None):
setup_ats()
elif name == 'historyserver':
setup_historyserver()
+ elif name == 'apptimelinereader':
+ if not params.use_external_hbase and not params.is_hbase_system_service_launch:
+ setup_atsv2_hbase_directories()
+ setup_atsv2_hbase_files()
XmlConfig("core-site.xml",
conf_dir=config_dir,
@@ -121,9 +133,13 @@ def yarn(name=None, config_dir=None):
mode=0o644
)
+
+ configs = {}
+ configs.update(params.config['configurations']['yarn-site'])
+ configs["hadoop.registry.dns.bind-port"] = params.config['configurations']['yarn-env']['registry.dns.bind-port']
XmlConfig("yarn-site.xml",
conf_dir=config_dir,
- configurations=params.config['configurations']['yarn-site'],
+ configurations=configs,
configuration_attributes=params.config['configurationAttributes']['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
@@ -138,6 +154,25 @@ def yarn(name=None, config_dir=None):
group=params.user_group,
mode=0o644
)
+
+ if not isinstance(params.hbase_site_conf, UnknownConfiguration):
+ XmlConfig("hbase-site.xml",
+ conf_dir=params.yarn_hbase_conf_dir,
+ configurations=params.hbase_site_conf,
+ configuration_attributes=params.hbase_site_attributes,
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ mode=0o644
+ )
+
+ XmlConfig("resource-types.xml",
+ conf_dir=config_dir,
+ configurations=params.config['configurations']['resource-types'],
+ configuration_attributes=params.config['configurationAttributes']['resource-types'],
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0o644
+ )
File(format("{limits_conf_dir}/yarn.conf"),
mode=0o644,
content=Template('yarn.conf.j2')
@@ -201,6 +236,14 @@ def yarn(name=None, config_dir=None):
group=params.user_group,
content=Template("yarn_ats_jaas.conf.j2")
)
+
+ if params.has_registry_dns:
+ File(os.path.join(config_dir, 'yarn_registry_dns_jaas.conf'),
+ owner=params.yarn_user,
+ group=params.user_group,
+ mode=0o644,
+ content=Template("yarn_registry_dns_jaas.conf.j2")
+ )
File(os.path.join(config_dir, 'yarn_nm_jaas.conf'),
owner=params.yarn_user,
group=params.user_group,
@@ -287,6 +330,8 @@ def yarn(name=None, config_dir=None):
group=params.user_group
)
+ setup_atsv2_backend(name, config_dir)
+
def setup_historyserver():
import params
@@ -530,6 +575,12 @@ def yarn(name = None):
owner=params.yarn_user,
mode='f'
)
+ XmlConfig("yarn-hbase-site.xml",
+ conf_dir=params.config_dir,
+ configurations=params.config['configurations']['yarn-hbase-site'],
+ owner=params.yarn_user,
+ mode='f'
+ )
if name in params.service_map:
service_name = params.service_map[name]
@@ -538,3 +589,193 @@ def yarn(name = None):
action="change_user",
username = params.yarn_user,
password = Script.get_password(params.yarn_user))
+
+def setup_atsv2_backend(name=None, config_dir=None):
+ import params
+ Logger.info(f"setup_atsv2_backend name:{name} {params.use_external_hbase} {params.hbase_within_cluster} {params.is_hbase_system_service_launch}")
+ if name == "apptimelinereader" and params.use_external_hbase and params.hbase_within_cluster:
+ createTables()
+
+ if not params.use_external_hbase and params.is_hbase_system_service_launch:
+ if name == 'resourcemanager':
+ setup_system_services(config_dir)
+ elif name == 'nodemanager':
+ setup_atsv2_hbase_files()
+
+def setup_atsv2_hbase_files():
+ import params
+ if 'yarn-hbase-policy' in params.config['configurations']:
+ XmlConfig( "hbase-policy.xml",
+ conf_dir = params.yarn_hbase_conf_dir,
+ configurations = params.config['configurations']['yarn-hbase-policy'],
+ configuration_attributes=params.config['configurationAttributes']['yarn-hbase-policy'],
+ owner = params.yarn_hbase_user,
+ group = params.user_group,
+ mode=0o644
+ )
+
+ File(os.path.join(params.yarn_hbase_conf_dir, "hbase-env.sh"),
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ mode=0o644,
+ content=InlineTemplate(params.yarn_hbase_env_sh_template)
+ )
+
+ File( format("{yarn_hbase_grant_premissions_file}"),
+ owner = params.yarn_hbase_user,
+ group = params.user_group,
+ mode = 0o644,
+ content = Template('yarn_hbase_grant_permissions.j2')
+ )
+
+ if (params.yarn_hbase_log4j_props != None):
+ File(format("{yarn_hbase_conf_dir}/log4j.properties"),
+ mode=0o644,
+ group=params.user_group,
+ owner=params.yarn_hbase_user,
+ content=InlineTemplate(params.yarn_hbase_log4j_props)
+ )
+ elif (os.path.exists(format("{yarn_hbase_conf_dir}/log4j.properties"))):
+ File(format("{yarn_hbase_conf_dir}/log4j.properties"),
+ mode=0o644,
+ group=params.user_group,
+ owner=params.yarn_hbase_user
+ )
+ if params.security_enabled:
+ File(os.path.join(params.yarn_hbase_conf_dir, 'yarn_hbase_master_jaas.conf'),
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ content=Template("yarn_hbase_master_jaas.conf.j2")
+ )
+ File(os.path.join(params.yarn_hbase_conf_dir, 'yarn_hbase_regionserver_jaas.conf'),
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ content=Template("yarn_hbase_regionserver_jaas.conf.j2")
+ )
+ # Metrics properties
+ if params.has_metric_collector:
+ File(os.path.join(params.yarn_hbase_conf_dir, 'hadoop-metrics2-hbase.properties'),
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ content=Template("hadoop-metrics2-hbase.properties.j2")
+ )
+
+def setup_atsv2_hbase_directories():
+ import params
+ Directory([params.yarn_hbase_pid_dir_prefix, params.yarn_hbase_pid_dir, params.yarn_hbase_log_dir],
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ create_parents=True,
+ cd_access='a',
+ )
+
+ parent_dir = os.path.dirname(params.yarn_hbase_tmp_dir)
+ # In case if we have several placeholders in path
+ while ("${" in parent_dir):
+ parent_dir = os.path.dirname(parent_dir)
+ if parent_dir != os.path.abspath(os.sep) :
+ Directory (parent_dir,
+ create_parents = True,
+ cd_access="a",
+ )
+ Execute(("chmod", "1777", parent_dir), sudo=True)
+
+def setup_system_services(config_dir=None):
+ import params
+ setup_atsv2_hbase_files()
+ if params.security_enabled:
+ File(os.path.join(params.yarn_hbase_conf_dir, 'hbase.yarnfile'),
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ content=Template("yarn_hbase_secure.yarnfile.j2")
+ )
+ else:
+ File(os.path.join(params.yarn_hbase_conf_dir, 'hbase.yarnfile'),
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ content=Template("yarn_hbase_unsecure.yarnfile.j2")
+ )
+
+
+ user_dir = format("{yarn_system_service_dir}/{yarn_system_service_launch_mode}/{yarn_hbase_user}")
+ params.HdfsResource(user_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.yarn_user,
+ group=params.user_group
+ )
+ params.HdfsResource(format("{user_dir}/hbase.yarnfile"),
+ type="file",
+ action="create_on_execute",
+ source=format("{yarn_hbase_conf_dir}/hbase.yarnfile"),
+ owner=params.yarn_user,
+ group=params.user_group
+ )
+ params.HdfsResource(format("{yarn_hbase_user_home}"),
+ type="directory",
+ action="create_on_execute",
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ mode=0o770,
+ )
+ params.HdfsResource(format("{yarn_hbase_user_version_home}"),
+ type="directory",
+ action="create_on_execute",
+ owner=params.yarn_hbase_user,
+ group=params.user_group,
+ mode=0o770,
+ )
+ params.HdfsResource(format("{yarn_hbase_user_version_home}/core-site.xml"),
+ type="file",
+ action="create_on_execute",
+ source=format("{config_dir}/core-site.xml"),
+ owner=params.yarn_hbase_user,
+ group=params.user_group
+ )
+ params.HdfsResource(format("{yarn_hbase_user_version_home}/hbase-site.xml"),
+ type="file",
+ action="create_on_execute",
+ source=format("{yarn_hbase_conf_dir}/hbase-site.xml"),
+ owner=params.yarn_hbase_user,
+ group=params.user_group
+ )
+ params.HdfsResource(format("{yarn_hbase_user_version_home}/hbase-policy.xml"),
+ type="file",
+ action="create_on_execute",
+ source=format("{yarn_hbase_conf_dir}/hbase-policy.xml"),
+ owner=params.yarn_hbase_user,
+ group=params.user_group
+ )
+ params.HdfsResource(format("{yarn_hbase_user_version_home}/log4j.properties"),
+ type="file",
+ action="create_on_execute",
+ source=format("{yarn_hbase_conf_dir}/log4j.properties"),
+ owner=params.yarn_hbase_user,
+ group=params.user_group
+ )
+ if params.has_metric_collector:
+ params.HdfsResource(format("{yarn_hbase_user_version_home}/hadoop-metrics2-hbase.properties"),
+ type="file",
+ action="create_on_execute",
+ source=format("{yarn_hbase_conf_dir}/hadoop-metrics2-hbase.properties"),
+ owner=params.yarn_hbase_user,
+ group=params.user_group
+ )
+ params.HdfsResource(params.yarn_hbase_hdfs_root_dir,
+ type="directory",
+ action="create_on_execute",
+ owner=params.yarn_hbase_user
+ )
+ # copy service-dep.tar.gz into hdfs
+ params.HdfsResource(format("{yarn_service_app_hdfs_path}"),
+ type="directory",
+ action="create_on_execute",
+ owner=params.hdfs_user,
+ group=params.hdfs_user,
+ mode=0o555,
+ )
+
+ params.HdfsResource(None, action="execute")
+
+ create_hbase_package()
+ copy_hbase_package_to_hdfs()
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/yarn_registry_dns.py
old mode 100644
new mode 100755
similarity index 66%
copy from ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py
copy to ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/yarn_registry_dns.py
index 4217990f95..9bd40252ee
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/scripts/yarn_registry_dns.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -20,86 +19,75 @@ Ambari Agent
"""
+import nodemanager_upgrade
+
from resource_management.libraries.script.script import Script
-from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions import stack_select
from resource_management.libraries.functions.constants import StackFeature
from resource_management.libraries.functions.stack_features import check_stack_feature
from resource_management.libraries.functions.check_process_status import check_process_status
-from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.security_commons import build_expectations, \
cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
FILE_TYPE_XML
-from resource_management.core.source import Template
from resource_management.core.logger import Logger
-
-from install_jars import install_tez_jars
from yarn import yarn
from service import service
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyImpl
-class HistoryServer(Script):
+class RegistryDNS(Script):
def install(self, env):
self.install_packages(env)
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
- service('historyserver', action='stop', serviceName='mapreduce')
+ service('registrydns',action='stop')
- def configure(self, env):
+ def start(self, env, upgrade_type=None):
import params
env.set_params(params)
- yarn(name="historyserver")
-
+ self.configure(env) # FOR SECURITY
+ service('registrydns',action='start')
-@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
-class HistoryserverWindows(HistoryServer):
- def start(self, env):
+ def configure(self, env):
import params
env.set_params(params)
- self.configure(env)
- service('historyserver', action='start', serviceName='mapreduce')
+ yarn(name="registrydns")
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class RegistryDNSWindows(RegistryDNS):
def status(self, env):
- service('historyserver', action='status')
+ service('registrydns', action='status')
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
-class HistoryServerDefault(HistoryServer):
+class RegistryDNSDefault(RegistryDNS):
def pre_upgrade_restart(self, env, upgrade_type=None):
- Logger.info("Executing Stack Upgrade pre-restart")
+ Logger.info("Executing RegistryDNS Stack Upgrade pre-restart")
import params
env.set_params(params)
-
- if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
- stack_select.select_packages(params.version)
-
- def start(self, env, upgrade_type=None):
- import params
- env.set_params(params)
- self.configure(env) # FOR SECURITY
-
- service('historyserver', action='start', serviceName='mapreduce')
+ stack_select.select_packages(params.version)
def status(self, env):
import status_params
env.set_params(status_params)
- check_process_status(status_params.mapred_historyserver_pid_file)
+ check_process_status(status_params.yarn_registry_dns_in_use_pid_file)
def get_log_folder(self):
import params
- return params.mapred_log_dir
-
+ return params.yarn_log_dir
+
def get_user(self):
import params
- return params.mapred_user
+ return params.yarn_user
def get_pid_files(self):
import status_params
- return [status_params.mapred_historyserver_pid_file]
+ return [status_params.yarn_registry_dns_in_use_pid_file]
if __name__ == "__main__":
- HistoryServer().execute()
+ RegistryDNS().execute()
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/hadoop-metrics2-hbase.properties.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/hadoop-metrics2-hbase.properties.j2
new file mode 100755
index 0000000000..5bb6d6f5fc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/hadoop-metrics2-hbase.properties.j2
@@ -0,0 +1,62 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+hbase.extendedperiod = 3600
+
+hbase.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.period=30
+hbase.collector.hosts={{ams_collector_hosts}}
+hbase.port={{metric_collector_port}}
+hbase.protocol={{metric_collector_protocol}}
+
+jvm.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+jvm.period=30
+jvm.collector.hosts={{ams_collector_hosts}}
+jvm.port={{metric_collector_port}}
+jvm.protocol={{metric_collector_protocol}}
+
+rpc.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+rpc.period=30
+rpc.collector.hosts={{ams_collector_hosts}}
+rpc.port={{metric_collector_port}}
+rpc.protocol={{metric_collector_protocol}}
+
+*.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar
+*.sink.timeline.slave.host.name={{hostname}}
+*.host_in_memory_aggregation = {{host_in_memory_aggregation}}
+*.host_in_memory_aggregation_port = {{host_in_memory_aggregation_port}}
+{% if is_aggregation_https_enabled %}
+*.host_in_memory_aggregation_protocol = {{host_in_memory_aggregation_protocol}}
+{% endif %}
+
+hbase.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink
+hbase.sink.timeline.period={{metrics_collection_period}}
+hbase.sink.timeline.sendInterval={{metrics_report_interval}}000
+hbase.sink.timeline.collector.hosts={{ams_collector_hosts}}
+hbase.sink.timeline.port={{metric_collector_port}}
+hbase.sink.timeline.protocol={{metric_collector_protocol}}
+hbase.sink.timeline.serviceName-prefix=yarn-timeline-storage
+
+# HTTPS properties
+hbase.sink.timeline.truststore.path = {{metric_truststore_path}}
+hbase.sink.timeline.truststore.type = {{metric_truststore_type}}
+hbase.sink.timeline.truststore.password = {{metric_truststore_password}}
+
+# Switch off metrics generation on a per region basis
+*.source.filter.class=org.apache.hadoop.metrics2.filter.RegexFilter
+hbase.*.source.filter.exclude=.*(Regions|Users|Tables).*
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/input.config-yarn.json.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/input.config-yarn.json.j2
new file mode 100755
index 0000000000..7d75279bc8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/input.config-yarn.json.j2
@@ -0,0 +1,86 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+ "input":[
+ {
+ "type":"yarn_nodemanager",
+ "rowtype":"service",
+ "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-nodemanager-*.log"
+ },
+ {
+ "type":"yarn_resourcemanager",
+ "rowtype":"service",
+ "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-resourcemanager-*.log"
+ },
+ {
+ "type":"yarn_timelineserver",
+ "rowtype":"service",
+ "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-timelineserver-*.log"
+ },
+ {
+ "type":"yarn_jobsummary",
+ "rowtype":"service",
+ "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-mapreduce.jobsummary.log"
+ }
+ ],
+ "filter":[
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "yarn_historyserver",
+ "yarn_nodemanager",
+ "yarn_resourcemanager",
+ "yarn_timelineserver"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ },
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "yarn_jobsummary"
+ ]
+ }
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "logtime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
old mode 100644
new mode 100755
index 55308e839e..fdebe402f1
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
@@ -15,6 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
+
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ storeKey=true
+ useTicketCache=false
+ keyTab="{{yarn_timelineservice_keytab}}"
+ principal="{{yarn_timelineservice_principal_name}}";
+};
com.sun.security.jgss.krb5.initiate {
com.sun.security.auth.module.Krb5LoginModule required
renewTGT=false
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_grant_permissions.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_grant_permissions.j2
new file mode 100755
index 0000000000..e464cf8ce8
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_grant_permissions.j2
@@ -0,0 +1,39 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+grant '{{yarn_user}}', '{{yarn_user_hbase_permissions}}'
+exit
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_master_jaas.conf.j2
old mode 100644
new mode 100755
similarity index 65%
copy from ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
copy to ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_master_jaas.conf.j2
index 55308e839e..5a92102baf
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_master_jaas.conf.j2
@@ -15,13 +15,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
+
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{yarn_hbase_master_keytab}}"
+principal="{{yarn_hbase_master_principal_name}}";
+};
com.sun.security.jgss.krb5.initiate {
- com.sun.security.auth.module.Krb5LoginModule required
- renewTGT=false
- doNotPrompt=true
- useKeyTab=true
- keyTab="{{yarn_timelineservice_keytab}}"
- principal="{{yarn_timelineservice_principal_name}}"
- storeKey=true
- useTicketCache=false;
+com.sun.security.auth.module.Krb5LoginModule required
+renewTGT=false
+doNotPrompt=true
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{yarn_hbase_master_keytab}}"
+principal="{{yarn_hbase_master_principal_name}}";
};
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_package_preparation.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_package_preparation.j2
new file mode 100755
index 0000000000..007c239e08
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_package_preparation.j2
@@ -0,0 +1,76 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+echo "`date` Running hbase package creation script"
+ambari_agent_tmp_dir="{{tmp_dir}}"
+yarn_hbase_user="{{yarn_hbase_user}}"
+yarn_hbase_user_tmp="{{yarn_hbase_user_tmp}}"
+version_home="{{stack_root}}/{{version}}/usr/lib"
+if [ ! -d $yarn_hbase_user_tmp ]
+then
+ echo "Creating a tmp path $yarn_hbase_user_tmp";
+ mkdir -p $yarn_hbase_user_tmp;
+ cd $ambari_agent_tmp_dir;
+ chmod -R 0755 $yarn_hbase_user;
+fi;
+
+cd $yarn_hbase_user_tmp
+if [ -f hbase.tar.gz ]
+then
+ echo "`date` hbase.tar.gz package exist in path $yarn_hbase_user_tmp."
+ echo "`date` File status: `ls -l hbase.tar.gz`"
+ echo "`date` Exitting from script.."
+ exit
+fi;
+
+cp -rf $version_home/hbase .
+rm -rf hbase/conf
+rm -rf hbase/logs
+rm -rf hbase/pids
+rm -rf hbase/lib/zookeeper*.jar
+cp -rf $version_home/zookeeper/zookeeper-*.jar hbase/lib
+cp -rf $version_home/hadoop/mapreduce.tar.gz .
+tar -xzf mapreduce.tar.gz
+rm -rf mapreduce.tar.gz
+tar -czf hbase.tar.gz hbase hadoop && echo "`date` HBase package created in path $yarn_hbase_user_tmp"
+chmod 644 hbase.tar.gz && echo "`date` hbase.tar.gz has set with ugo=644"
+rm -rf hbase
+rm -rf hadoop
+echo "`date` File status: `ls -l $yarn_hbase_user_tmp/hbase.tar.gz`"
+echo "`date` Exitting from script.."
+exit
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_regionserver_jaas.conf.j2
old mode 100644
new mode 100755
similarity index 64%
copy from ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
copy to ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_regionserver_jaas.conf.j2
index 55308e839e..0ec41f5b73
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_regionserver_jaas.conf.j2
@@ -15,13 +15,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
+
+Client {
+com.sun.security.auth.module.Krb5LoginModule required
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{yarn_hbase_regionserver_keytab}}"
+principal="{{yarn_hbase_regionserver_principal_name}}";
+};
com.sun.security.jgss.krb5.initiate {
- com.sun.security.auth.module.Krb5LoginModule required
- renewTGT=false
- doNotPrompt=true
- useKeyTab=true
- keyTab="{{yarn_timelineservice_keytab}}"
- principal="{{yarn_timelineservice_principal_name}}"
- storeKey=true
- useTicketCache=false;
+com.sun.security.auth.module.Krb5LoginModule required
+renewTGT=false
+doNotPrompt=true
+useKeyTab=true
+storeKey=true
+useTicketCache=false
+keyTab="{{yarn_hbase_regionserver_keytab}}"
+principal="{{yarn_hbase_regionserver_principal_name}}";
};
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_secure.yarnfile.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_secure.yarnfile.j2
new file mode 100755
index 0000000000..c4a3fc34ad
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_secure.yarnfile.j2
@@ -0,0 +1,172 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{
+ "name": "ats-hbase",
+ "version": "1.0.0",
+ "lifetime": "-1",
+ "queue": "{{yarn_hbase_service_queue_name}}",
+ "artifact": {
+ "id": "{{yarn_hbase_app_hdfs_path}}/hbase.tar.gz",
+ "type": "TARBALL"
+ },
+ "configuration": {
+ "properties" : {
+ "yarn.service.container-failure.retry.max" : 10,
+ "yarn.service.framework.path" : "{{yarn_service_app_hdfs_path}}/service-dep.tar.gz"
+ },
+ "env": {
+ "HBASE_IDENT_STRING": "${USER}",
+ "HBASE_LOG_DIR": "<LOG_DIR>",
+ "HBASE_CONF_DIR": "$PWD/conf",
+ "HBASE_PID_DIR": "$PWD/pids",
+ "JAVA_HOME": "{{java64_home}}",
+ "HBASE_ROOT_LOGGER": "{{yarn_hbase_log_level}},RFA",
+ "SERVER_GC_OPTS": "-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${HBASE_LOG_DIR}/gc.log-`date +'%Y%m%d%H%M'`",
+ "HBASE_OPTS": "-XX:+UseConcMarkSweepGC -XX:ErrorFile=${HBASE_LOG_DIR}/hs_err_pid%p.log -Djava.io.tmpdir=/tmp",
+ "HADOOP_HOME": "$PWD/lib/hadoop",
+ "HADOOP_YARN_HOME": "$PWD/lib/hadoop",
+ "HBASE_HOME": "$PWD/lib/hbase"
+ },
+ "files": [
+ {
+ "type": "TEMPLATE",
+ "dest_file": "log4j.properties",
+ "src_file": "{{yarn_hbase_user_version_home}}/log4j.properties"
+ },
+ {
+ "type": "TEMPLATE",
+ "dest_file": "hbase-site.xml",
+ "src_file": "{{yarn_hbase_user_version_home}}/hbase-site.xml"
+ },
+ {
+ "type": "TEMPLATE",
+ "dest_file": "hbase-policy.xml",
+ "src_file": "{{yarn_hbase_user_version_home}}/hbase-policy.xml"
+ },
+ {
+ "type": "TEMPLATE",
+ "dest_file": "core-site.xml",
+ "src_file": "{{yarn_hbase_user_version_home}}/core-site.xml"
+ },
+ {
+ "type": "TEMPLATE",
+ "dest_file": "hadoop-metrics2-hbase.properties",
+ "src_file": "{{yarn_hbase_user_version_home}}/hadoop-metrics2-hbase.properties"
+ }
+ ]
+ },
+ "components": [
+ {
+ "name": "master",
+ "number_of_containers": {{yarn_hbase_master_containers}},
+ "launch_command": "$HBASE_HOME/bin/hbase master start",
+ "readiness_check": {
+ "type": "HTTP",
+ "properties": {
+ "url": "http://${THIS_HOST}:{{yarn_hbase_master_info_port}}/master-status"
+ }
+ },
+ "resource": {
+ "cpus": {{yarn_hbase_master_cpu}},
+ "memory": "{{yarn_hbase_master_memory}}"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_PREFIX": "hbase-$HBASE_IDENT_STRING-master-$HOSTNAME",
+ "HBASE_LOGFILE": "$HBASE_LOG_PREFIX.log",
+ "HBASE_MASTER_OPTS": "-Xms{{yarn_hbase_master_heapsize}} -Xmx{{yarn_hbase_master_heapsize}} -Djava.security.auth.login.config={{yarn_hbase_master_jaas_file}}"
+ }
+ },
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "node",
+ "target_tags": [
+ "master"
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": {{yarn_hbase_regionserver_containers}},
+ "launch_command": "$HBASE_HOME/bin/hbase regionserver start",
+ "readiness_check": {
+ "type": "HTTP",
+ "properties": {
+ "url": "http://${THIS_HOST}:{{yarn_hbase_regionserver_info_port}}/rs-status"
+ }
+ },
+ "resource": {
+ "cpus": {{yarn_hbase_regionserver_cpu}},
+ "memory": "{{yarn_hbase_regionserver_memory}}"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_PREFIX": "hbase-$HBASE_IDENT_STRING-regionserver-$HOSTNAME",
+ "HBASE_LOGFILE": "$HBASE_LOG_PREFIX.log",
+ "HBASE_REGIONSERVER_OPTS": "-XX:CMSInitiatingOccupancyFraction=70 -XX:ReservedCodeCacheSize=256m -Xms{{yarn_hbase_regionserver_heapsize}} -Xmx{{yarn_hbase_regionserver_heapsize}} -Djava.security.auth.login.config={{yarn_hbase_regionserver_jaas_file}}"
+ }
+ },
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "node",
+ "target_tags": [
+ "regionserver"
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "name": "hbaseclient",
+ "number_of_containers": {{yarn_hbase_client_containers}},
+ "launch_command": "sleep 10;export HBASE_CLASSPATH_PREFIX=$HADOOP_HOME/share/hadoop/yarn/timelineservice/*;{{yarn_hbase_kinit_cmd}} $HBASE_HOME/bin/hbase {{class_name}}; {{yarn_hbase_kinit_cmd}} $HBASE_HOME/bin/hbase shell {{yarn_hbase_grant_premissions_file}};sleep infinity",
+ "dependencies": [ "master", "regionserver" ],
+ "resource": {
+ "cpus": {{yarn_hbase_client_cpu}},
+ "memory": "{{yarn_hbase_client_memory}}"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_PREFIX": "hbase-$HBASE_IDENT_STRING-hbaseclient-$HOSTNAME",
+ "HBASE_LOGFILE": "$HBASE_LOG_PREFIX.log"
+ }
+ },
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "node",
+ "target_tags": [
+ "hbaseclient"
+ ]
+ }
+ ]
+ }
+ }
+ ],
+ "kerberos_principal" : {
+ "principal_name" : "{{yarn_ats_hbase_principal_name}}",
+ "keytab" : "file://{{yarn_ats_hbase_keytab}}"
+ }
+}
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_unsecure.yarnfile.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_unsecure.yarnfile.j2
new file mode 100755
index 0000000000..1bc4155970
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_hbase_unsecure.yarnfile.j2
@@ -0,0 +1,168 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+{
+ "name": "ats-hbase",
+ "version": "1.0.0",
+ "lifetime": "-1",
+ "queue": "{{yarn_hbase_service_queue_name}}",
+ "artifact": {
+ "id": "{{yarn_hbase_app_hdfs_path}}/hbase.tar.gz",
+ "type": "TARBALL"
+ },
+ "configuration": {
+ "properties" : {
+ "yarn.service.container-failure.retry.max" : 10,
+ "yarn.service.framework.path" : "{{yarn_service_app_hdfs_path}}/service-dep.tar.gz"
+ },
+ "env": {
+ "HBASE_IDENT_STRING": "${USER}",
+ "HBASE_LOG_DIR": "<LOG_DIR>",
+ "HBASE_CONF_DIR": "$PWD/conf",
+ "HBASE_PID_DIR": "$PWD/pids",
+ "JAVA_HOME": "{{java64_home}}",
+ "HBASE_ROOT_LOGGER": "{{yarn_hbase_log_level}},RFA",
+ "SERVER_GC_OPTS": "-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:${HBASE_LOG_DIR}/gc.log-`date +'%Y%m%d%H%M'`",
+ "HBASE_OPTS": "-XX:+UseConcMarkSweepGC -XX:ErrorFile=${HBASE_LOG_DIR}/hs_err_pid%p.log -Djava.io.tmpdir=/tmp",
+ "HADOOP_HOME": "$PWD/lib/hadoop",
+ "HADOOP_YARN_HOME": "$PWD/lib/hadoop",
+ "HBASE_HOME": "$PWD/lib/hbase"
+ },
+ "files": [
+ {
+ "type": "TEMPLATE",
+ "dest_file": "log4j.properties",
+ "src_file": "{{yarn_hbase_user_version_home}}/log4j.properties"
+ },
+ {
+ "type": "TEMPLATE",
+ "dest_file": "hbase-site.xml",
+ "src_file": "{{yarn_hbase_user_version_home}}/hbase-site.xml"
+ },
+ {
+ "type": "TEMPLATE",
+ "dest_file": "hbase-policy.xml",
+ "src_file": "{{yarn_hbase_user_version_home}}/hbase-policy.xml"
+ },
+ {
+ "type": "TEMPLATE",
+ "dest_file": "core-site.xml",
+ "src_file": "{{yarn_hbase_user_version_home}}/core-site.xml"
+ },
+ {
+ "type": "TEMPLATE",
+ "dest_file": "hadoop-metrics2-hbase.properties",
+ "src_file": "{{yarn_hbase_user_version_home}}/hadoop-metrics2-hbase.properties"
+ }
+ ]
+ },
+ "components": [
+ {
+ "name": "master",
+ "number_of_containers": {{yarn_hbase_master_containers}},
+ "launch_command": "$HBASE_HOME/bin/hbase master start",
+ "readiness_check": {
+ "type": "HTTP",
+ "properties": {
+ "url": "http://${THIS_HOST}:{{yarn_hbase_master_info_port}}/master-status"
+ }
+ },
+ "resource": {
+ "cpus": {{yarn_hbase_master_cpu}},
+ "memory": "{{yarn_hbase_master_memory}}"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_PREFIX": "hbase-$HBASE_IDENT_STRING-master-$HOSTNAME",
+ "HBASE_LOGFILE": "$HBASE_LOG_PREFIX.log",
+ "HBASE_MASTER_OPTS": "-Xms{{yarn_hbase_master_heapsize}} -Xmx{{yarn_hbase_master_heapsize}}"
+ }
+ },
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "node",
+ "target_tags": [
+ "master"
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "name": "regionserver",
+ "number_of_containers": {{yarn_hbase_regionserver_containers}},
+ "launch_command": "$HBASE_HOME/bin/hbase regionserver start",
+ "readiness_check": {
+ "type": "HTTP",
+ "properties": {
+ "url": "http://${THIS_HOST}:{{yarn_hbase_regionserver_info_port}}/rs-status"
+ }
+ },
+ "resource": {
+ "cpus": {{yarn_hbase_regionserver_cpu}},
+ "memory": "{{yarn_hbase_regionserver_memory}}"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_PREFIX": "hbase-$HBASE_IDENT_STRING-regionserver-$HOSTNAME",
+ "HBASE_LOGFILE": "$HBASE_LOG_PREFIX.log",
+ "HBASE_REGIONSERVER_OPTS": "-XX:CMSInitiatingOccupancyFraction=70 -XX:ReservedCodeCacheSize=256m -Xms{{yarn_hbase_regionserver_heapsize}} -Xmx{{yarn_hbase_regionserver_heapsize}}"
+ }
+ },
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "node",
+ "target_tags": [
+ "regionserver"
+ ]
+ }
+ ]
+ }
+ },
+ {
+ "name": "hbaseclient",
+ "number_of_containers": {{yarn_hbase_client_containers}},
+ "launch_command": "sleep 10;export HBASE_CLASSPATH_PREFIX=$HADOOP_HOME/share/hadoop/yarn/timelineservice/*;$HBASE_HOME/bin/hbase {{class_name}};sleep infinity",
+ "dependencies": [ "master", "regionserver" ],
+ "resource": {
+ "cpus": {{yarn_hbase_client_cpu}},
+ "memory": "{{yarn_hbase_client_memory}}"
+ },
+ "configuration": {
+ "env": {
+ "HBASE_LOG_PREFIX": "hbase-$HBASE_IDENT_STRING-hbaseclient-$HOSTNAME",
+ "HBASE_LOGFILE": "$HBASE_LOG_PREFIX.log"
+ }
+ },
+ "placement_policy": {
+ "constraints": [
+ {
+ "type": "ANTI_AFFINITY",
+ "scope": "node",
+ "target_tags": [
+ "hbaseclient"
+ ]
+ }
+ ]
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_nm_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_nm_jaas.conf.j2
index b501c82415..0a51afdf57 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_nm_jaas.conf.j2
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_nm_jaas.conf.j2
@@ -15,6 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
+
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ useKeyTab=true
+ storeKey=true
+ useTicketCache=false
+ keyTab="{{nodemanager_keytab}}"
+ principal="{{nodemanager_principal_name}}";
+};
com.sun.security.jgss.krb5.initiate {
com.sun.security.auth.module.Krb5LoginModule required
renewTGT=false
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_registry_dns_jaas.conf.j2
old mode 100644
new mode 100755
similarity index 74%
copy from ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
copy to ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_registry_dns_jaas.conf.j2
index 55308e839e..bec53cb0e5
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_ats_jaas.conf.j2
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/package/templates/yarn_registry_dns_jaas.conf.j2
@@ -15,13 +15,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#}
+
+Client {
+ com.sun.security.auth.module.Krb5LoginModule required
+ doNotPrompt=true
+ useKeyTab=true
+ storeKey=true
+ useTicketCache=false
+ keyTab="{{yarn_registry_dns_keytab}}"
+ principal="{{yarn_registry_dns_principal_name}}";
+};
com.sun.security.jgss.krb5.initiate {
com.sun.security.auth.module.Krb5LoginModule required
renewTGT=false
doNotPrompt=true
useKeyTab=true
- keyTab="{{yarn_timelineservice_keytab}}"
- principal="{{yarn_timelineservice_principal_name}}"
+ keyTab="{{yarn_registry_dns_keytab}}"
+ principal="{{yarn_registry_dns_principal_name}}"
storeKey=true
useTicketCache=false;
};
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/properties/container-executor.cfg.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/properties/container-executor.cfg.j2
index c6f1ff676d..42e7d58c8b 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/properties/container-executor.cfg.j2
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/properties/container-executor.cfg.j2
@@ -38,3 +38,22 @@ yarn.nodemanager.log-dirs={{nm_log_dirs}}
yarn.nodemanager.linux-container-executor.group={{yarn_executor_container_group}}
banned.users=hdfs,yarn,mapred,bin
min.user.id={{min_user_id}}
+
+{{ '[docker]' }}
+ module.enabled={{docker_module_enabled}}
+ docker.binary={{docker_binary}}
+ docker.allowed.capabilities={{docker_allowed_capabilities}}
+ docker.allowed.devices={{docker_allowed_devices}}
+ docker.allowed.networks={{docker_allowed_networks}}
+ docker.allowed.ro-mounts={{nm_local_dirs}},{{docker_allowed_ro_mounts}}
+ docker.allowed.rw-mounts={{nm_local_dirs}},{{nm_log_dirs}},{{docker_allowed_rw_mounts}}
+ docker.privileged-containers.enabled={{docker_privileged_containers_enabled}}
+ docker.trusted.registries={{docker_trusted_registries}}
+ docker.allowed.volume-drivers={{docker_allowed_volume_drivers}}
+
+{{ '[gpu]' }}
+ module.enabled={{gpu_module_enabled}}
+
+{{ '[cgroups]' }}
+ root={{cgroup_root}}
+ yarn-hierarchy={{yarn_hierarchy}}
\ No newline at end of file
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/quicklinks/quicklinks.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/quicklinks/quicklinks.json
index ced9385287..9b25fa56fd 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/quicklinks/quicklinks.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/quicklinks/quicklinks.json
@@ -20,7 +20,7 @@
"label": "ResourceManager UI",
"requires_user_name": "false",
"component_name": "RESOURCEMANAGER",
- "url": "%@://%@:%@",
+ "url": "%@://%@:%@/ui2",
"port":{
"http_property": "yarn.resourcemanager.webapp.address",
"http_default_port": "8088",
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/role_command_order.json b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/role_command_order.json
index 9ab3b4828d..af5451046e 100644
--- a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/YARN/role_command_order.json
@@ -11,8 +11,8 @@
"HISTORYSERVER-START": ["NAMENODE-START", "DATANODE-START"],
"HISTORYSERVER-RESTART": ["NAMENODE-RESTART"],
"YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
- "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"]
-
+ "MAPREDUCE_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
+ "TIMELINE_READER-START": ["ZOOKEEPER_SERVER-START", "NAMENODE-START", "DATANODE-START","HBASE_REGIONSERVER-START"]
},
"_comment" : "Dependencies that are used in ResourceManager HA cluster",
"resourcemanager_optional_ha" : {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ambari.apache.org
For additional commands, e-mail: commits-help@ambari.apache.org