You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2017/07/31 14:10:37 UTC
[22/25] ambari git commit: AMBARI-21610. EU: Modify configs to enable
custom extensions functionality. (swagle)
AMBARI-21610. EU: Modify configs to enable custom extensions functionality. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5aa398c6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5aa398c6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5aa398c6
Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 5aa398c631808c08ab433bc34a2adda1e6cb6666
Parents: 9b309e3
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Fri Jul 28 17:04:58 2017 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Fri Jul 28 17:47:45 2017 -0700
----------------------------------------------------------------------
.../4.2.5/upgrades/config-upgrade.xml | 24 ++++++++++++++++++
.../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 22 ++++++++++++++++-
.../BigInsights/4.2/upgrades/config-upgrade.xml | 26 ++++++++++++++++++++
.../upgrades/nonrolling-upgrade-to-hdp-2.6.xml | 22 ++++++++++++++++-
4 files changed, 92 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
index f71e91a..f4068a2 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
@@ -39,6 +39,7 @@
<type>hadoop-env</type>
<replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64" replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native" />
<replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}" replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} if [ -d "/usr/lib/hadoop-lzo" ]; then export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/* export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native fi"/>
+ <insert key="content" value="{% if hadoop_custom_extensions_enabled %} export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/* {% endif %}" insert-type="append" newline-after="true" newline-before="true"/>
</definition>
<definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments">
<type>hdfs-site</type>
@@ -51,6 +52,10 @@
<type>hadoop-env</type>
<insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
</definition>
+ <definition xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+ <type>core-site</type>
+ <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop" if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/>
+ </definition>
</changes>
</component>
</service>
@@ -99,6 +104,14 @@
<type>hbase-env</type>
<set key="backup_wal_dir" value="true"/>
</definition>
+ <definition xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions" summary="Enable custom extension">
+ <type>hbase-site</type>
+ <set key="hbase.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hbase" if-type="hbase-site" if-key-state="absent" if-key="hbase.custom-extensions.root"/>
+ </definition>
+ <definition xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions" summary="Enable custom extensions">
+ <type>hbase-env</type>
+ <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements. Optional. export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*" newline-after="true" newline-before="true"/>
+ </definition>
</changes>
</component>
</service>
@@ -216,6 +229,17 @@
<type>hive-site</type>
<set key="custom.hive.warehouse.mode" value="0770" if-type="hive-site" if-key="custom.hive.warehouse.mode" if-key-state="absent"/>
</definition>
+
+ <definition xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions" summary="Enable custom extensions in hive-site">
+ <type>hive-site</type>
+ <set key="hive.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hive" if-key-state="absent" if-type="hive-site" if-key="hive.custom-extensions.root"/>
+ </definition>
+
+ <definition xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions" summary="Enable custom extensions in hive-env">
+ <type>hive-env</type>
+ <replace key="content" find="if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then"/>
+ <replace key="content" find="if [ -f "${HIVE_AUX_JARS_PATH}" ]; then" replace-with="if [ -f "${HIVE_AUX_JARS_PATH}" ] || [ -d "${HIVE_AUX_JARS_PATH}" ] ; then"/>
+ </definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index a7fdeec..00e8b22 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -222,6 +222,12 @@
</task>
</execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Enable custom extensions in core-site">
+ <task xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+ <summary>Enable custom extensions in core-site</summary>
+ </task>
+ </execute-stage>
+
<!-- core-site -->
<execute-stage title="Fix auth_to_local mapping">
<condition xsi:type="security" type="kerberos"/>
@@ -257,6 +263,14 @@
<task xsi:type="configure" id="biginsights_4_2_hbase_env_config" />
</execute-stage>
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE">
+ <task xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"/>
+ </execute-stage>
+
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE">
+ <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/>
+ </execute-stage>
+
<!--RANGER-->
<execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger Admin">
<task xsi:type="configure" id="hdp_2_6_0_0_remove_bind_anonymous"/>
@@ -304,7 +318,13 @@
</execute-stage>
<execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="biginsights_4_2_hive_server_configure_authentication"/>
- </execute-stage>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server">
+ <task xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"/>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server">
+ <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/>
+ </execute-stage>
<execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for Hive WebHCat server">
<task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations" />
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
index e8ccc22..17e80b5 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
@@ -131,6 +131,7 @@
<type>hadoop-env</type>
<replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64" replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native" />
<replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}" replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS} if [ -d "/usr/lib/hadoop-lzo" ]; then export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/* export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native fi"/>
+ <insert key="content" value="{% if hadoop_custom_extensions_enabled %} export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/* {% endif %}" insert-type="append" newline-after="true" newline-before="true"/>
</definition>
<definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments">
<type>hdfs-site</type>
@@ -143,6 +144,10 @@
<type>hadoop-env</type>
<insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
</definition>
+ <definition xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+ <type>core-site</type>
+ <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop" if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/>
+ </definition>
</changes>
</component>
</service>
@@ -153,6 +158,7 @@
<definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update Yarn configurations">
<type>yarn-site</type>
<replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/>
+ <insert key="yarn.application.classpath" insert-type="append" newline-after="false" newline-before="false" value=",/usr/hdp/current/ext/hadoop/*"/>
</definition>
<definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
<type>yarn-env</type>
@@ -168,6 +174,7 @@
<definition xsi:type="configure" id="biginsights_4_2_mapreduce_application_framework_patch" summary="Update MapReduce2 configurations">
<type>mapred-site</type>
<set key="mapreduce.application.framework.path" value="/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework"/>
+ <insert key="mapreduce.application.classpath" insert-type="append" value=":/usr/hdp/current/ext/hadoop/*" newline-before="false" newline-after="false"/>
</definition>
</changes>
</component>
@@ -191,6 +198,14 @@
<type>hbase-env</type>
<set key="backup_wal_dir" value="true"/>
</definition>
+ <definition xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions" summary="Enable custom extensions in hbase-site">
+ <type>hbase-site</type>
+ <set key="hbase.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hbase" if-type="hbase-site" if-key-state="absent" if-key="hbase.custom-extensions.root"/>
+ </definition>
+ <definition xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions" summary="Enable custom extensions in hbase-env">
+ <type>hbase-env</type>
+ <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements. Optional. export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*" newline-after="true" newline-before="true"/>
+ </definition>
</changes>
</component>
</service>
@@ -240,6 +255,17 @@
<type>hive-site</type>
<set key="custom.hive.warehouse.mode" value="0770" if-type="hive-site" if-key="custom.hive.warehouse.mode" if-key-state="absent"/>
</definition>
+
+ <definition xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions" summary="Enable custom extensions in hive-site">
+ <type>hive-site</type>
+ <set key="hive.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hive" if-key-state="absent" if-type="hive-site" if-key="hive.custom-extensions.root"/>
+ </definition>
+
+ <definition xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions" summary="Enable custom extensions in hive-env">
+ <type>hive-env</type>
+ <replace key="content" find="if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive if [ "${HIVE_AUX_JARS_PATH}" != "" ]; then"/>
+ <replace key="content" find="if [ -f "${HIVE_AUX_JARS_PATH}" ]; then" replace-with="if [ -f "${HIVE_AUX_JARS_PATH}" ] || [ -d "${HIVE_AUX_JARS_PATH}" ] ; then"/>
+ </definition>
</changes>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index ed6414d..55736cf 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -217,6 +217,12 @@
</task>
</execute-stage>
+ <execute-stage service="HDFS" component="NAMENODE" title="Enable custom extensions in core-site">
+ <task xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+ <summary>Enable custom extensions in core-site</summary>
+ </task>
+ </execute-stage>
+
<!-- core-site -->
<execute-stage title="Fix auth_to_local mapping">
<condition xsi:type="security" type="kerberos"/>
@@ -252,6 +258,14 @@
<task xsi:type="configure" id="biginsights_4_2_hbase_env_config" />
</execute-stage>
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE">
+ <task xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"/>
+ </execute-stage>
+
+ <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE">
+ <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/>
+ </execute-stage>
+
<!-- HIVE -->
<execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="biginsights_4_2_hive_env_configure"/>
@@ -261,7 +275,13 @@
</execute-stage>
<execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
<task xsi:type="configure" id="biginsights_4_2_hive_server_configure_authentication"/>
- </execute-stage>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server">
+ <task xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"/>
+ </execute-stage>
+ <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server">
+ <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/>
+ </execute-stage>
<execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for Hive WebHCat server">
<task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations" />
</execute-stage>