You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2017/02/07 13:38:03 UTC
ambari git commit: AMBARI-19879. Updating yarn-env and hadoop-env
templates with ZK secure options on stack upgrade (Attila Magyar via
magyari_sandor)
Repository: ambari
Updated Branches:
refs/heads/trunk 9b8055cc4 -> 3561cf7be
AMBARI-19879. Updating yarn-env and hadoop-env templates with ZK secure options on stack upgrade (Attila Magyar via magyari_sandor)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3561cf7b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3561cf7b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3561cf7b
Branch: refs/heads/trunk
Commit: 3561cf7bef0c0ca55ca1b178168bf6a441a8f835
Parents: 9b8055c
Author: Attila Magyar <am...@hortonworks.com>
Authored: Tue Feb 7 14:31:01 2017 +0100
Committer: Sandor Magyari <sm...@hortonworks.com>
Committed: Tue Feb 7 14:31:01 2017 +0100
----------------------------------------------------------------------
.../HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py | 3 +--
.../HDFS/3.0.0.3.0/package/scripts/zkfc_slave.py | 1 +
.../stacks/HDP/2.3/upgrades/config-upgrade.xml | 9 +++++++++
.../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 12 ++++++++++++
.../resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 2 ++
.../stacks/HDP/2.4/upgrades/config-upgrade.xml | 8 ++++++++
.../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 14 ++++++++++++++
.../resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 2 ++
.../stacks/HDP/2.5/upgrades/config-upgrade.xml | 8 ++++++++
.../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 13 +++++++++++++
.../resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 2 ++
11 files changed, 72 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
index c2ff457..be0d2ed 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
@@ -39,8 +39,6 @@ from resource_management.libraries.functions.stack_features import check_stack_f
from resource_management.libraries.script import Script
from resource_management.core.resources.zkmigrator import ZkMigrator
-
-
class ZkfcSlave(Script):
def get_component_name(self):
import params
@@ -62,6 +60,7 @@ class ZkfcSlave(Script):
import params
env.set_params(params)
hdfs("zkfc_slave")
+ utils.set_up_zkfc_security(params)
pass
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/zkfc_slave.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/zkfc_slave.py
index 6a0d71f..19a78c7 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/zkfc_slave.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/zkfc_slave.py
@@ -60,6 +60,7 @@ class ZkfcSlave(Script):
import params
env.set_params(params)
hdfs("zkfc_slave")
+ utils.set_up_zkfc_security(params)
pass
@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index f96b246..0277a4e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -375,6 +375,11 @@
<regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
<regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
</definition>
+
+ <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs">
+ <type>hadoop-env</type>
+ <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+ </definition>
</changes>
</component>
</service>
@@ -502,6 +507,10 @@
<regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
<regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
</definition>
+ <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
+ <type>yarn-env</type>
+ <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+ </definition>
</changes>
</component>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index 2bb2a72..8da11ff 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -286,6 +286,12 @@
</task>
</execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs">
+ <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+ <summary>Adding HDFS ZKFC Security ACLs</summary>
+ </task>
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig">
@@ -293,6 +299,12 @@
</task>
</execute-stage>
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs">
+ <task xsi:type="configure" id="yarn_env_security_opts">
+ <summary>Adding YARN Security ACLs</summary>
+ </task>
+ </execute-stage>
+
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for Yarn Resourcemanager">
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index 290d3c5..01fc102 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -680,6 +680,7 @@
<task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" />
<task xsi:type="configure" id="hdfs_log4j_parameterize" />
+ <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -760,6 +761,7 @@
</task>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" />
<task xsi:type="configure" id="yarn_log4j_parameterize" />
+ <task xsi:type="configure" id="yarn_env_security_opts" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index 4c17177..7cb727f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -264,6 +264,10 @@
<regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
<regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
</definition>
+ <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs">
+ <type>hadoop-env</type>
+ <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+ </definition>
</changes>
</component>
</service>
@@ -289,6 +293,10 @@
<regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
<regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
</definition>
+ <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
+ <type>yarn-env</type>
+ <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+ </definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
index ea3de50..046904b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
@@ -281,6 +281,13 @@
</task>
</execute-stage>
+ <!--Yarn-->
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs">
+ <task xsi:type="configure" id="yarn_env_security_opts">
+ <summary>Adding YARN Security ACLs</summary>
+ </task>
+ </execute-stage>
+
<!-- YARN -->
<execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
<task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
@@ -400,6 +407,13 @@
</task>
</execute-stage>
+ <!--HDFS-->
+ <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs">
+ <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+ <summary>Adding HDFS ZKFC Security ACLs</summary>
+ </task>
+ </execute-stage>
+
<!-- SQOOP -->
<execute-stage service="SQOOP" component="SQOOP" title="Apply config changes for Sqoop to remove Atlas Configs">
<!-- Remove Atlas configs that were incorrectly added to sqoop-site instead of Atlas' application.properties. -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index 5371d3e..70bb2ca 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -685,6 +685,7 @@
<task xsi:type="configure" id="hdp_2_5_0_0_namenode_ha_adjustments"/>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" />
<task xsi:type="configure" id="hdfs_log4j_parameterize" />
+ <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
@@ -765,6 +766,7 @@
</task>
<task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" />
<task xsi:type="configure" id="yarn_log4j_parameterize" />
+ <task xsi:type="configure" id="yarn_env_security_opts" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 188e6f4..54a824d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -128,6 +128,10 @@
<regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
<regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
</definition>
+ <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
+ <type>yarn-env</type>
+ <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+ </definition>
</changes>
</component>
</service>
@@ -146,6 +150,10 @@
<regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
<regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
</definition>
+ <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs">
+ <type>hadoop-env</type>
+ <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
+ </definition>
</changes>
</component>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index b975eba..5b8351b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -291,6 +291,13 @@
</task>
</execute-stage>
+ <!--Yarn-->
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs">
+ <task xsi:type="configure" id="yarn_env_security_opts">
+ <summary>Adding YARN Security ACLs</summary>
+ </task>
+ </execute-stage>
+
<!--TEZ-->
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez">
<task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath">
@@ -317,6 +324,12 @@
</task>
</execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs">
+ <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+ <summary>Adding HDFS ZKFC Security ACLs</summary>
+ </task>
+ </execute-stage>
+
<!--HBASE-->
<execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties">
<task xsi:type="configure" id="hbase_log4j_parameterize">
http://git-wip-us.apache.org/repos/asf/ambari/blob/3561cf7b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 291397a..2f07c97 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -589,6 +589,7 @@
<component name="NAMENODE">
<pre-upgrade>
<task xsi:type="configure" id="hdfs_log4j_parameterize" />
+ <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
</pre-upgrade>
<pre-downgrade />
<upgrade>
@@ -667,6 +668,7 @@
<component name="RESOURCEMANAGER">
<pre-upgrade>
<task xsi:type="configure" id="yarn_log4j_parameterize" />
+ <task xsi:type="configure" id="yarn_env_security_opts" />
</pre-upgrade>
<pre-downgrade />
<upgrade>